Added fallback support for latin1 file encoding for data files. An UTF-8 BOM is requi...

git-svn-id: https://casparcg.svn.sourceforge.net/svnroot/casparcg/server/trunk@3207 362d55ac-95cf-4e76-9f9a-cbaa9c17b72d

diff --git a/protocol/amcp/AMCPCommandsImpl.cpp b/protocol/amcp/AMCPCommandsImpl.cpp
index 91df432f5b48b7b71ec6e57b507c1888c7e9e7e7..21c08a0faf557711f4d677e8b5c84d6ff9ab7cfa 100644 (file)
--- a/protocol/amcp/AMCPCommandsImpl.cpp
+++ b/protocol/amcp/AMCPCommandsImpl.cpp
@@ -35,6 +35,7 @@
 #include <common/os/windows/current_version.h>\r
 #include <common/os/windows/system_info.h>\r
 #include <common/utility/string.h>\r
+#include <common/utility/utf8conv.h>\r
 \r
 #include <core/producer/frame_producer.h>\r
 #include <core/video_format.h>\r
@@ -61,6 +62,7 @@
 #include <algorithm>\r
 #include <locale>\r
 #include <fstream>\r
+#include <memory>\r
 #include <cctype>\r
 #include <io.h>\r
 \r
@@ -68,8 +70,12 @@
 #include <boost/lexical_cast.hpp>\r
 #include <boost/algorithm/string.hpp>\r
 #include <boost/filesystem.hpp>\r
+#include <boost/filesystem/fstream.hpp>\r
 #include <boost/regex.hpp>\r
 #include <boost/property_tree/xml_parser.hpp>\r
+#include <boost/locale.hpp>\r
+#include <boost/range/adaptor/transformed.hpp>\r
+#include <boost/range/algorithm/copy.hpp>\r
 \r
 #include <tbb/concurrent_unordered_map.h>\r
 \r
@@ -99,6 +105,76 @@ namespace caspar { namespace protocol {
 \r
 using namespace core;\r
 \r
+std::wstring read_utf8_file(const boost::filesystem::wpath& file)\r
+{\r
+       std::wstringstream result;\r
+       boost::filesystem::wifstream filestream(file);\r
+\r
+       if (filestream) \r
+       {\r
+               // Consume BOM first\r
+               filestream.get();\r
+               // read all data\r
+               result << filestream.rdbuf();\r
+       }\r
+\r
+       return result.str();\r
+}\r
+\r
+std::wstring read_latin1_file(const boost::filesystem::wpath& file)\r
+{\r
+       boost::locale::generator gen;\r
+       gen.locale_cache_enabled(true);\r
+       gen.categories(boost::locale::codepage_facet);\r
+\r
+       std::stringstream result_stream;\r
+       boost::filesystem::ifstream filestream(file);\r
+       filestream.imbue(gen("en_US.ISO8859-1"));\r
+\r
+       if (filestream)\r
+       {\r
+               // read all data\r
+               result_stream << filestream.rdbuf();\r
+       }\r
+\r
+       std::string result = result_stream.str();\r
+       std::wstring widened_result;\r
+\r
+       // The first 255 codepoints in unicode is the same as in latin1\r
+       auto from_signed_to_signed = std::function<unsigned char(char)>(\r
+               [] (char c) { return static_cast<unsigned char>(c); }\r
+       );\r
+       boost::copy(\r
+               result | boost::adaptors::transformed(from_signed_to_signed),\r
+               std::back_inserter(widened_result));\r
+\r
+       return widened_result;\r
+}\r
+\r
+std::wstring read_file(const boost::filesystem::wpath& file)\r
+{\r
+       static const uint8_t BOM[] = {0xef, 0xbb, 0xbf};\r
+\r
+       if (!boost::filesystem::exists(file))\r
+       {\r
+               return L"";\r
+       }\r
+\r
+       if (boost::filesystem::file_size(file) >= 3)\r
+       {\r
+               boost::filesystem::ifstream bom_stream(file);\r
+\r
+               char header[3];\r
+               bom_stream.read(header, 3);\r
+               bom_stream.close();\r
+\r
+               if (std::memcmp(BOM, header, 3) == 0)\r
+                       return read_utf8_file(file);\r
+       }\r
+\r
+       return read_latin1_file(file);\r
+}\r
+\r
 std::wstring MediaInfo(const boost::filesystem::wpath& path)\r
 {\r
        if(boost::filesystem::is_regular_file(path))\r
@@ -979,19 +1055,8 @@ bool CGCommand::DoExecuteAdd() {
                        filename.append(dataString);\r
                        filename.append(TEXT(".ftd"));\r
 \r
-                       //open file\r
-                       std::wifstream datafile(filename.c_str());\r
-\r
-                       if(datafile) \r
-                       {\r
-                               //read all data\r
-                               data << datafile.rdbuf();\r
-                               datafile.close();\r
-\r
-                               //extract data to _parameters\r
-                               dataFromFile = data.str();\r
-                               pDataString = dataFromFile.c_str();\r
-                       }\r
+                       dataFromFile = read_file(boost::filesystem::wpath(filename));\r
+                       pDataString = dataFromFile.c_str();\r
                }\r
        }\r
 \r
@@ -1128,19 +1193,7 @@ bool CGCommand::DoExecuteUpdate()
                        filename.append(dataString);\r
                        filename.append(TEXT(".ftd"));\r
 \r
-                       //open file\r
-                       std::wifstream datafile(filename.c_str());\r
-\r
-                       if(datafile) \r
-                       {\r
-                               std::wstringstream data;\r
-                               //read all data\r
-                               data << datafile.rdbuf();\r
-                               datafile.close();\r
-\r
-                               //extract data to _parameters\r
-                               dataString = data.str();\r
-                       }\r
+                       dataString = read_file(boost::filesystem::wpath(filename));\r
                }               \r
 \r
                int layer = _ttoi(_parameters.at(1).c_str());\r
@@ -1244,7 +1297,8 @@ bool DataCommand::DoExecuteStore()
                return false;\r
        }\r
 \r
-       datafile << _parameters2[2];\r
+       datafile << static_cast<wchar_t>(65279); // UTF-8 BOM character\r
+       datafile << _parameters2[2] << std::flush;\r
        datafile.close();\r
 \r
        std::wstring replyString = TEXT("202 DATA STORE OK\r\n");\r
@@ -1264,16 +1318,18 @@ bool DataCommand::DoExecuteRetrieve()
        filename.append(_parameters[1]);\r
        filename.append(TEXT(".ftd"));\r
 \r
-       std::wifstream datafile(filename.c_str());\r
+       std::wstring file_contents = read_file(boost::filesystem::wpath(filename));\r
 \r
-       if(!datafile) \r
+       //std::wifstream datafile(filename.c_str());\r
+\r
+       if (file_contents.empty()) \r
        {\r
                SetReplyString(TEXT("404 DATA RETRIEVE ERROR\r\n"));\r
                return false;\r
        }\r
 \r
        std::wstringstream reply(TEXT("201 DATA RETRIEVE OK\r\n"));\r
-       std::wstring line;\r
+       /*std::wstring line;\r
        bool bFirstLine = true;\r
        while(std::getline(datafile, line))\r
        {\r
@@ -1284,8 +1340,9 @@ bool DataCommand::DoExecuteRetrieve()
 \r
                reply << line;\r
        }\r
-       datafile.close();\r
+       datafile.close();*/\r
 \r
+       reply << file_contents;\r
        reply << "\r\n";\r
        SetReplyString(reply.str());\r
        return true;\r