Merged r270:HEAD of the emitting-unicode branch

include/emitter.h

@@ -29,6 +29,7 @@ namespace YAML
 		const std::string GetLastError() const;
 		
 		// global setters
+		bool SetOutputCharset(EMITTER_MANIP value);
 		bool SetStringFormat(EMITTER_MANIP value);
 		bool SetBoolFormat(EMITTER_MANIP value);
 		bool SetIntBase(EMITTER_MANIP value);

include/emittermanip.h

@@ -11,6 +11,10 @@ namespace YAML
 	enum EMITTER_MANIP {
 		// general manipulators
 		Auto,
+
+		// output character set
+		EmitNonAscii,
+		EscapeNonAscii,
 		
 		// string manipulators
 		// Auto, // duplicate

src/emitter.cpp

@@ -37,6 +37,11 @@ namespace YAML
 	}
 
 	// global setters
+	bool Emitter::SetOutputCharset(EMITTER_MANIP value)
+	{
+		return m_pState->SetOutputCharset(value, GLOBAL);
+	}
+
 	bool Emitter::SetStringFormat(EMITTER_MANIP value)
 	{
 		return m_pState->SetStringFormat(value, GLOBAL);
@@ -485,13 +490,14 @@ namespace YAML
 		PreAtomicWrite();
 		EmitSeparationIfNecessary();
 		
+		bool escapeNonAscii = m_pState->GetOutputCharset() == EscapeNonAscii;
 		EMITTER_MANIP strFmt = m_pState->GetStringFormat();
 		FLOW_TYPE flowType = m_pState->GetCurGroupFlowType();
 		unsigned curIndent = m_pState->GetCurIndent();
 
 		switch(strFmt) {
 			case Auto:
-				Utils::WriteString(m_stream, str, flowType == FT_FLOW);
+				Utils::WriteString(m_stream, str, flowType == FT_FLOW, escapeNonAscii);
 				break;
 			case SingleQuoted:
 				if(!Utils::WriteSingleQuotedString(m_stream, str)) {
@@ -500,11 +506,11 @@ namespace YAML
 				}
 				break;
 			case DoubleQuoted:
-				Utils::WriteDoubleQuotedString(m_stream, str);
+				Utils::WriteDoubleQuotedString(m_stream, str, escapeNonAscii);
 				break;
 			case Literal:
 				if(flowType == FT_FLOW)
-					Utils::WriteString(m_stream, str, flowType == FT_FLOW);
+					Utils::WriteString(m_stream, str, flowType == FT_FLOW, escapeNonAscii);
 				else
 					Utils::WriteLiteralString(m_stream, str, curIndent + m_pState->GetIndent());
 				break;

src/emitterstate.cpp

@@ -9,6 +9,7 @@ namespace YAML
 		m_stateStack.push(ES_WAITING_FOR_DOC);
 		
 		// set default global manipulators
+		m_charset.set(EmitNonAscii);
 		m_strFmt.set(Auto);
 		m_boolFmt.set(TrueFalseBool);
 		m_boolLengthFmt.set(LongBool);
@@ -43,6 +44,7 @@ namespace YAML
 	// . Only the ones that make sense will be accepted
 	void EmitterState::SetLocalValue(EMITTER_MANIP value)
 	{
+		SetOutputCharset(value, LOCAL);
 		SetStringFormat(value, LOCAL);
 		SetBoolFormat(value, LOCAL);
 		SetBoolCaseFormat(value, LOCAL);
@@ -132,6 +134,18 @@ namespace YAML
 	{
 		m_modifiedSettings.clear();
 	}
+
+	bool EmitterState::SetOutputCharset(EMITTER_MANIP value, FMT_SCOPE scope)
+	{
+		switch(value) {
+			case EmitNonAscii:
+			case EscapeNonAscii:
+				_Set(m_charset, value, scope);
+				return true;
+			default:
+				return false;
+		}
+	}
 	
 	bool EmitterState::SetStringFormat(EMITTER_MANIP value, FMT_SCOPE scope)
 	{

src/emitterstate.h

@@ -108,6 +108,9 @@ namespace YAML
 		void ClearModifiedSettings();
 
 		// formatters
+		bool SetOutputCharset(EMITTER_MANIP value, FMT_SCOPE scope);
+		EMITTER_MANIP GetOutputCharset() const { return m_charset.get(); }
+
 		bool SetStringFormat(EMITTER_MANIP value, FMT_SCOPE scope);
 		EMITTER_MANIP GetStringFormat() const { return m_strFmt.get(); }
 		
@@ -149,6 +152,7 @@ namespace YAML
 		// other state
 		std::stack <EMITTER_STATE> m_stateStack;
 		
+		Setting <EMITTER_MANIP> m_charset;
 		Setting <EMITTER_MANIP> m_strFmt;
 		Setting <EMITTER_MANIP> m_boolFmt;
 		Setting <EMITTER_MANIP> m_boolLengthFmt;

src/emitterutils.h

@@ -11,9 +11,9 @@ namespace YAML
 {
 	namespace Utils
 	{
-		bool WriteString(ostream& out, const std::string& str, bool inFlow);
+		bool WriteString(ostream& out, const std::string& str, bool inFlow, bool escapeNonAscii);
 		bool WriteSingleQuotedString(ostream& out, const std::string& str);
-		bool WriteDoubleQuotedString(ostream& out, const std::string& str);
+		bool WriteDoubleQuotedString(ostream& out, const std::string& str, bool escapeNonAscii);
 		bool WriteLiteralString(ostream& out, const std::string& str, int indent);
 		bool WriteComment(ostream& out, const std::string& str, int postCommentIndent);
 		bool WriteAlias(ostream& out, const std::string& str);

src/exp.cpp

@@ -28,9 +28,9 @@ namespace YAML
 			return value;
 		}
 
-		std::string Str(char ch)
+		std::string Str(unsigned ch)
 		{
-			return std::string("") + ch;
+			return std::string("") + static_cast<char>(ch);
 		}
 
 		// Escape

src/exp.h

@@ -26,7 +26,12 @@ namespace YAML
 		const RegEx Alpha = RegEx('a', 'z') || RegEx('A', 'Z');
 		const RegEx AlphaNumeric = Alpha || Digit;
 		const RegEx Hex = Digit || RegEx('A', 'F') || RegEx('a', 'f');
-		const RegEx Printable = RegEx(0x20, 0x7E);
+		// Valid Unicode code points that are not part of c-printable (YAML 1.2, sec. 5.1)
+		const RegEx NotPrintable = RegEx(0) || 
+			RegEx("\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x7F", REGEX_OR) || 
+			RegEx(0x0E, 0x1F) ||
+			(RegEx('\xC2') + (RegEx('\x80', '\x84') || RegEx('\x86', '\x9F')));
+		const RegEx Utf8_ByteOrderMark = RegEx("\xEF\xBB\xBF");
 
 		// actual tags
 

yaml-reader/emittertests.cpp

@@ -448,12 +448,25 @@ namespace Test
 			desiredOutput = "- ~\n-\n  null value: ~\n  ~: null key";
 		}
 		
-		void Unicode(YAML::Emitter& out, std::string& desiredOutput)
+		void EscapedUnicode(YAML::Emitter& out, std::string& desiredOutput)
 		{
-			out << "\x24 \xC2\xA2 \xE2\x82\xAC \xF0\xA4\xAD\xA2";
+			out << YAML::EscapeNonAscii << "\x24 \xC2\xA2 \xE2\x82\xAC \xF0\xA4\xAD\xA2";
 			
 			desiredOutput = "\"$ \\xa2 \\u20ac \\U00024b62\"";
 		}
+		
+		void Unicode(YAML::Emitter& out, std::string& desiredOutput)
+		{
+			out << "\x24 \xC2\xA2 \xE2\x82\xAC \xF0\xA4\xAD\xA2";
+			desiredOutput = "\x24 \xC2\xA2 \xE2\x82\xAC \xF0\xA4\xAD\xA2";
+		}
+		
+		void DoubleQuotedUnicode(YAML::Emitter& out, std::string& desiredOutput)
+		{
+			out << YAML::DoubleQuoted << "\x24 \xC2\xA2 \xE2\x82\xAC \xF0\xA4\xAD\xA2";
+			desiredOutput = "\"\x24 \xC2\xA2 \xE2\x82\xAC \xF0\xA4\xAD\xA2\"";
+		}
+		
 
 		////////////////////////////////////////////////////////////////////////////////////////////////////////
 		// incorrect emitting
@@ -616,7 +629,9 @@ namespace Test
 		RunEmitterTest(&Emitter::SimpleGlobalSettings, "simple global settings", passed, total);
 		RunEmitterTest(&Emitter::ComplexGlobalSettings, "complex global settings", passed, total);
 		RunEmitterTest(&Emitter::Null, "null", passed, total);
+		RunEmitterTest(&Emitter::EscapedUnicode, "escaped unicode", passed, total);
 		RunEmitterTest(&Emitter::Unicode, "unicode", passed, total);
+		RunEmitterTest(&Emitter::DoubleQuotedUnicode, "double quoted unicode", passed, total);
 		
 		RunEmitterErrorTest(&Emitter::ExtraEndSeq, "extra EndSeq", passed, total);
 		RunEmitterErrorTest(&Emitter::ExtraEndMap, "extra EndMap", passed, total);