From a4973b871125c04af61dc5b9420456ff0bb75fdb Mon Sep 17 00:00:00 2001
From: hellgore <hellgore@362d55ac-95cf-4e76-9f9a-cbaa9c17b72d>
Date: Tue, 4 Sep 2012 14:11:46 +0000
Subject: [PATCH] Merged latin1 fallback support to 2.1

git-svn-id: https://casparcg.svn.sourceforge.net/svnroot/casparcg/server/branches/2.1.0@3230 362d55ac-95cf-4e76-9f9a-cbaa9c17b72d
---
 protocol/amcp/AMCPCommandsImpl.cpp | 123 ++++++++++++++++++++---------
 1 file changed, 84 insertions(+), 39 deletions(-)

diff --git a/protocol/amcp/AMCPCommandsImpl.cpp b/protocol/amcp/AMCPCommandsImpl.cpp
index 01567a43b..38211a84d 100644
--- a/protocol/amcp/AMCPCommandsImpl.cpp
+++ b/protocol/amcp/AMCPCommandsImpl.cpp
@@ -60,6 +60,7 @@
 #include <algorithm>
 #include <locale>
 #include <fstream>
+#include <memory>
 #include <cctype>
 #include <io.h>
 
@@ -67,8 +68,12 @@
 #include <boost/lexical_cast.hpp>
 #include <boost/algorithm/string.hpp>
 #include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
 #include <boost/regex.hpp>
 #include <boost/property_tree/xml_parser.hpp>
+#include <boost/locale.hpp>
+#include <boost/range/adaptor/transformed.hpp>
+#include <boost/range/algorithm/copy.hpp>
 
 #include <tbb/concurrent_unordered_map.h>
 
@@ -98,6 +103,76 @@ namespace caspar { namespace protocol {
 
 using namespace core;
 
+std::wstring read_utf8_file(const boost::filesystem::wpath& file)
+{
+	std::wstringstream result;
+	boost::filesystem::wifstream filestream(file);
+
+	if (filestream) 
+	{
+		// Consume BOM first
+		filestream.get();
+		// read all data
+		result << filestream.rdbuf();
+	}
+
+	return result.str();
+}
+
+std::wstring read_latin1_file(const boost::filesystem::wpath& file)
+{
+	boost::locale::generator gen;
+	gen.locale_cache_enabled(true);
+	gen.categories(boost::locale::codepage_facet);
+
+	std::stringstream result_stream;
+	boost::filesystem::ifstream filestream(file);
+	filestream.imbue(gen("en_US.ISO8859-1"));
+
+	if (filestream)
+	{
+		// read all data
+		result_stream << filestream.rdbuf();
+	}
+
+	std::string result = result_stream.str();
+	std::wstring widened_result;
+
+	// The first 255 codepoints in unicode is the same as in latin1
+	auto from_signed_to_signed = std::function<unsigned char(char)>(
+		[] (char c) { return static_cast<unsigned char>(c); }
+	);
+	boost::copy(
+		result | boost::adaptors::transformed(from_signed_to_signed),
+		std::back_inserter(widened_result));
+
+	return widened_result;
+}
+
+std::wstring read_file(const boost::filesystem::wpath& file)
+{
+	static const uint8_t BOM[] = {0xef, 0xbb, 0xbf};
+
+	if (!boost::filesystem::exists(file))
+	{
+		return L"";
+	}
+
+	if (boost::filesystem::file_size(file) >= 3)
+	{
+		boost::filesystem::ifstream bom_stream(file);
+
+		char header[3];
+		bom_stream.read(header, 3);
+		bom_stream.close();
+
+		if (std::memcmp(BOM, header, 3) == 0)
+			return read_utf8_file(file);
+	}
+
+	return read_latin1_file(file);
+}
+
 std::wstring MediaInfo(const boost::filesystem::path& path)
 {
 	if(boost::filesystem::is_regular_file(path))
@@ -980,18 +1055,8 @@ bool CGCommand::DoExecuteAdd() {
 			filename.append(dataString);
 			filename.append(TEXT(".ftd"));
 
-			//open file
-			std::wifstream datafile(filename.c_str());
-			if(datafile) 
-			{
-				//read all data
-				data << datafile.rdbuf();
-				datafile.close();
-
-				//extract data to _parameters
-				dataFromFile = data.str();
-				pDataString = dataFromFile.c_str();
-			}
+			dataFromFile = read_file(boost::filesystem::wpath(filename));
+			pDataString = dataFromFile.c_str();
 		}
 	}
 
@@ -1128,18 +1193,7 @@ bool CGCommand::DoExecuteUpdate()
 			filename.append(dataString);
 			filename.append(TEXT(".ftd"));
 
-			//open file
-			std::wifstream datafile(filename.c_str());
-			if(datafile) 
-			{
-				std::wstringstream data;
-				//read all data
-				data << datafile.rdbuf();
-				datafile.close();
-
-				//extract data to _parameters
-				dataString = data.str();
-			}
+			dataString = read_file(boost::filesystem::wpath(filename));
 		}		
 
 		int layer = _ttoi(_parameters.at(1).c_str());
@@ -1242,7 +1296,8 @@ bool DataCommand::DoExecuteStore()
 		return false;
 	}
 
-	datafile << _parameters2[2];
+	datafile << static_cast<wchar_t>(65279); // UTF-8 BOM character
+	datafile << _parameters2[2] << std::flush;
 	datafile.close();
 
 	std::wstring replyString = TEXT("202 DATA STORE OK\r\n");
@@ -1262,27 +1317,17 @@ bool DataCommand::DoExecuteRetrieve()
 	filename.append(_parameters[1]);
 	filename.append(TEXT(".ftd"));
 
-	std::wifstream datafile(filename.c_str());
-	if(!datafile) 
+	std::wstring file_contents = read_file(boost::filesystem::wpath(filename));
+
+	if (file_contents.empty()) 
 	{
 		SetReplyString(TEXT("404 DATA RETRIEVE ERROR\r\n"));
 		return false;
 	}
 
 	std::wstringstream reply(TEXT("201 DATA RETRIEVE OK\r\n"));
-	std::wstring line;
-	bool bFirstLine = true;
-	while(std::getline(datafile, line))
-	{
-		if(!bFirstLine)
-			reply << "\\n";
-		else
-			bFirstLine = false;
-
-		reply << line;
-	}
-	datafile.close();
 
+	reply << file_contents;
 	reply << "\r\n";
 	SetReplyString(reply.str());
 	return true;
-- 
2.39.2