Read/Write data files in UTF-8, characters logged are replaced by ? if non-ascii...

git-svn-id: https://casparcg.svn.sourceforge.net/svnroot/casparcg/server/trunk@3106 362d55ac-95cf-4e76-9f9a-cbaa9c17b72d

diff --git a/common/log/log.cpp b/common/log/log.cpp
index e067d6ef744c8e3d7d6e00855c3fe23553844182..5060c559e049f8bfdec326d5a9020b29b0936604 100644 (file)
--- a/common/log/log.cpp
+++ b/common/log/log.cpp
@@ -63,12 +63,13 @@
 #include <boost/log/utility/init/common_attributes.hpp>\r
 #include <boost/log/utility/empty_deleter.hpp>\r
 #include <boost/lambda/lambda.hpp>\r
+#include <boost/bind.hpp>\r
 \r
 namespace caspar { namespace log {\r
 \r
 using namespace boost;\r
 \r
-void my_formatter(std::wostream& strm, boost::log::basic_record<wchar_t> const& rec)\r
+void my_formatter(bool print_all_characters, std::wostream& strm, boost::log::basic_record<wchar_t> const& rec)\r
 {\r
     namespace lambda = boost::lambda;\r
        \r
@@ -96,7 +97,14 @@ void my_formatter(std::wostream& strm, boost::log::basic_record<wchar_t> const&
                        strm << L" ";\r
        }\r
 \r
-    strm << replace_nonprintable_copy(rec.message(), L'?');\r
+       if (print_all_characters)\r
+       {\r
+               strm << rec.message();\r
+       }\r
+       else\r
+       {\r
+           strm << replace_nonprintable_copy(rec.message(), L'?');\r
+       }\r
 }\r
 \r
 namespace internal{\r
@@ -122,7 +130,7 @@ void init()
 //     stream_sink->set_filter(boost::log::filters::attr<severity_level>(boost::log::sources::aux::severity_attribute_name<wchar_t>::get()) >= debug);\r
 //#endif\r
 \r
-       stream_sink->locked_backend()->set_formatter(&my_formatter);\r
+       stream_sink->locked_backend()->set_formatter(boost::bind(my_formatter, false, _1, _2));\r
 \r
        boost::log::wcore::get()->add_sink(stream_sink);\r
 }\r
@@ -147,8 +155,13 @@ void add_file_sink(const std::wstring& folder)
                        boost::log::keywords::auto_flush = true,\r
                        boost::log::keywords::open_mode = std::ios::app\r
                );\r
-               \r
-               file_sink->locked_backend()->set_formatter(&my_formatter);\r
+\r
+               // TODO: does not seem to affect the character set only date formatting\r
+               file_sink->locked_backend()->imbue(caspar::get_narrow_locale());\r
+               // TODO: when UTF-8 log file output is fixed we can enable this\r
+               bool print_all_characters = false;\r
+\r
+               file_sink->locked_backend()->set_formatter(boost::bind(my_formatter, print_all_characters, _1, _2));\r
 \r
 //#ifdef NDEBUG\r
 //             file_sink->set_filter(boost::log::filters::attr<severity_level>(boost::log::sources::aux::severity_attribute_name<wchar_t>::get()) >= debug);\r

diff --git a/common/log/log.h b/common/log/log.h
index 2a86a090b5d556a38b0a46849723e27590b36d38..39b615dd64c7d0cbfac45f95cd7b22653ee3d571 100644 (file)
--- a/common/log/log.h
+++ b/common/log/log.h
@@ -102,7 +102,13 @@ template<typename T>
 inline void replace_nonprintable(std::basic_string<T, std::char_traits<T>, std::allocator<T>>& str, T with)\r
 {\r
        std::locale loc;\r
-       std::replace_if(str.begin(), str.end(), [&](T c)->bool { return !std::isprint(c, loc) && c != '\r' && c != '\n'; }, with);\r
+       std::replace_if(str.begin(), str.end(), [&](T c)->bool {\r
+               return \r
+                       (!std::isprint(c, loc) \r
+                       && c != '\r' \r
+                       && c != '\n')\r
+                       || c > static_cast<T>(127);\r
+       }, with);\r
 }\r
 \r
 template<typename T>\r

diff --git a/common/utility/string.cpp b/common/utility/string.cpp
index 76954b89b7c5c056c1fc6b23fdc752f5aa9434f3..1aef547b48865e97a378dcfe49e314066e35ef24 100644 (file)
--- a/common/utility/string.cpp
+++ b/common/utility/string.cpp
@@ -21,6 +21,8 @@
 \r
 #include "../stdafx.h"\r
 \r
+#include <boost/locale.hpp>\r
+\r
 #include "utf8conv.h"\r
 \r
 namespace caspar {\r
@@ -45,4 +47,14 @@ std::string narrow(const std::string& str)
        return str ;\r
 }\r
 \r
+std::locale get_narrow_locale()\r
+{\r
+       boost::locale::generator gen;\r
+\r
+       gen.locale_cache_enabled(true);\r
+\r
+       // TODO: make configurable?\r
+       return gen.generate("en_GB.UTF-8");\r
+}\r
+\r
 }
\ No newline at end of file

diff --git a/common/utility/string.h b/common/utility/string.h
index a7aee7a8195d7f90f31a067defe26b91deedb16c..29fff2b0773823b1b2b7f13e3c9712e465f6b630 100644 (file)
--- a/common/utility/string.h
+++ b/common/utility/string.h
@@ -22,6 +22,7 @@
 #pragma once\r
 \r
 #include <string>\r
+#include <locale>\r
 #include <boost/lexical_cast.hpp>\r
           \r
 namespace caspar {\r
@@ -30,6 +31,7 @@ std::wstring widen(const std::string& str);
 std::wstring widen(const std::wstring& str);\r
 std::string narrow(const std::wstring& str);      \r
 std::string narrow(const std::string& str);\r
+std::locale get_narrow_locale();\r
 \r
 template <typename T>\r
 inline T lexical_cast_or_default(const std::wstring str, T fail_value = T())\r

diff --git a/protocol/amcp/AMCPCommandsImpl.cpp b/protocol/amcp/AMCPCommandsImpl.cpp
index 9ef4c5c42702de51097a89120d8ae8f67d4c41df..f64e893848aa4284083fea8071c2f4a3f1d6b92f 100644 (file)
--- a/protocol/amcp/AMCPCommandsImpl.cpp
+++ b/protocol/amcp/AMCPCommandsImpl.cpp
@@ -34,6 +34,7 @@
 #include <common/diagnostics/graph.h>\r
 #include <common/os/windows/current_version.h>\r
 #include <common/os/windows/system_info.h>\r
+#include <common/utility/string.h>\r
 \r
 #include <core/producer/frame_producer.h>\r
 #include <core/video_format.h>\r
@@ -980,6 +981,8 @@ bool CGCommand::DoExecuteAdd() {
 \r
                        //open file\r
                        std::wifstream datafile(filename.c_str());\r
+                       datafile.imbue(caspar::get_narrow_locale());\r
+\r
                        if(datafile) \r
                        {\r
                                //read all data\r
@@ -1128,6 +1131,8 @@ bool CGCommand::DoExecuteUpdate()
 \r
                        //open file\r
                        std::wifstream datafile(filename.c_str());\r
+                       datafile.imbue(caspar::get_narrow_locale());\r
+\r
                        if(datafile) \r
                        {\r
                                std::wstringstream data;\r
@@ -1233,7 +1238,11 @@ bool DataCommand::DoExecuteStore()
        filename.append(_parameters[1]);\r
        filename.append(TEXT(".ftd"));\r
 \r
-       std::wofstream datafile(filename.c_str());\r
+       std::wofstream datafile;\r
+\r
+       datafile.imbue(caspar::get_narrow_locale());\r
+       datafile.open(filename.c_str());\r
+\r
        if(!datafile) \r
        {\r
                SetReplyString(TEXT("501 DATA STORE FAILED\r\n"));\r
@@ -1261,6 +1270,9 @@ bool DataCommand::DoExecuteRetrieve()
        filename.append(TEXT(".ftd"));\r
 \r
        std::wifstream datafile(filename.c_str());\r
+\r
+       datafile.imbue(caspar::get_narrow_locale());\r
+\r
        if(!datafile) \r
        {\r
                SetReplyString(TEXT("404 DATA RETRIEVE ERROR\r\n"));\r

diff --git a/protocol/amcp/AMCPProtocolStrategy.cpp b/protocol/amcp/AMCPProtocolStrategy.cpp
index f51f2da206cff299191d524786f90d91881c15a3..b94a4e4f18443447e99dc135c234485225288310 100644 (file)
--- a/protocol/amcp/AMCPProtocolStrategy.cpp
+++ b/protocol/amcp/AMCPProtocolStrategy.cpp
@@ -112,7 +112,7 @@ void AMCPProtocolStrategy::Parse(const TCHAR* pData, int charCount, ClientInfoPt
 \r
 void AMCPProtocolStrategy::ProcessMessage(const std::wstring& message, ClientInfoPtr& pClientInfo)\r
 {      \r
-       CASPAR_LOG(info) << L"Received message from " << pClientInfo->print() << ": " << log::replace_nonprintable_copy(message, L'?') + L"\\r\\n";\r
+       CASPAR_LOG(info) << L"Received message from " << pClientInfo->print() << ": " << message << L"\\r\\n";\r
        \r
        bool bError = true;\r
        MessageParserState state = New;\r

diff --git a/protocol/cii/CIIProtocolStrategy.cpp b/protocol/cii/CIIProtocolStrategy.cpp
index 3fb2ff320c33228db5437da3a3475ed4351dc43c..9e6769dbb8505c81618c77bc0fd56271ad46fc2e 100644 (file)
--- a/protocol/cii/CIIProtocolStrategy.cpp
+++ b/protocol/cii/CIIProtocolStrategy.cpp
@@ -85,7 +85,7 @@ void CIIProtocolStrategy::Parse(const TCHAR* pData, int charCount, IO::ClientInf
 \r
 void CIIProtocolStrategy::ProcessMessage(const std::wstring& message, IO::ClientInfoPtr pClientInfo)\r
 {      \r
-       CASPAR_LOG(info) << L"Received message from " << pClientInfo->print() << ": " << log::replace_nonprintable_copy(message, L'?') + L"\\r\\n";\r
+       CASPAR_LOG(info) << L"Received message from " << pClientInfo->print() << ": " << message << L"\\r\\n";\r
 \r
        std::vector<std::wstring> tokens;\r
        int tokenCount = TokenizeMessage(message, &tokens);\r

diff --git a/protocol/util/ClientInfo.h b/protocol/util/ClientInfo.h
index c7fd4285dad8f6db200f428dece2a54ea06cd235..77c7e58dae96d4f37357c6bfce042946152e3213 100644 (file)
--- a/protocol/util/ClientInfo.h
+++ b/protocol/util/ClientInfo.h
@@ -25,6 +25,8 @@
 #include <string>\r
 #include <iostream>\r
 \r
+#include <common/log/log.h>\r
+\r
 namespace caspar { namespace IO {\r
 \r
 class ClientInfo \r
@@ -47,7 +49,7 @@ struct ConsoleClientInfo : public caspar::IO::ClientInfo
 {\r
        void Send(const std::wstring& data)\r
        {\r
-               std::wcout << (L"#" + data);\r
+               std::wcout << (L"#" + caspar::log::replace_nonprintable_copy(data, L'?'));\r
        }\r
        void Disconnect(){}\r
        virtual std::wstring print() const {return L"Console";}\r