Merged utf branch changes r178:187 into the trunk

CMakeLists.txt

@@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 2.6)
 
 project (YAML_CPP)
 
+SET(CMAKE_CXX_FLAGS "-O2")
+
 enable_testing()
 
 if(WIN32)
@@ -36,4 +38,4 @@ install(
 )
 
 add_subdirectory (yaml-reader)
-
+add_subdirectory (util)

src/emitterutils.cpp

@@ -3,6 +3,7 @@
 #include "indentation.h"
 #include "exceptions.h"
 #include <sstream>
+#include "stringsource.h"
 
 namespace YAML
 {
@@ -29,8 +30,8 @@ namespace YAML
 				                          || (!Exp::Printable)
 				                          || Exp::Break
 				                          || Exp::Tab;
-				Buffer buffer(&str[0], str.size());
-				while(buffer.size) {
+				StringCharSource buffer(str.c_str(), str.size());
+				while(buffer) {
 					if(disallowed.Matches(buffer))
 						return false;
 					++buffer;

src/regex.cpp

 #include "crt.h"
 #include "regex.h"
-#include "stream.h"
-#include <iostream>
 
 namespace YAML
 {
-	RegEx::RegEx(REGEX_OP op): m_op(op), m_pOp(0)
+	// constructors
+	RegEx::RegEx(): m_op(REGEX_EMPTY)
 	{
-		SetOp();
 	}
-
-	RegEx::RegEx(const RegEx& rhs): m_pOp(0)
-	{
-		m_op = rhs.m_op;
-		m_a = rhs.m_a;
-		m_z = rhs.m_z;
-		m_params = rhs.m_params;
-
-		SetOp();
-	}
-
-	RegEx::RegEx(): m_op(REGEX_EMPTY), m_pOp(0)
+	
+	RegEx::RegEx(REGEX_OP op): m_op(op)
 	{
-		SetOp();
 	}
-
-	RegEx::RegEx(char ch): m_op(REGEX_MATCH), m_pOp(0), m_a(ch)
+	
+	RegEx::RegEx(char ch): m_op(REGEX_MATCH), m_a(ch)
 	{
-		SetOp();
 	}
-
-	RegEx::RegEx(char a, char z): m_op(REGEX_RANGE), m_pOp(0), m_a(a), m_z(z)
+	
+	RegEx::RegEx(char a, char z): m_op(REGEX_RANGE), m_a(a), m_z(z)
 	{
-		SetOp();
 	}
-
-	RegEx::RegEx(const std::string& str, REGEX_OP op): m_op(op), m_pOp(0)
+	
+	RegEx::RegEx(const std::string& str, REGEX_OP op): m_op(op)
 	{
 		for(unsigned i=0;i<str.size();i++)
 			m_params.push_back(RegEx(str[i]));
-
-		SetOp();
-	}
-
-	RegEx::~RegEx()
-	{
-		delete m_pOp;
-	}
-
-	RegEx& RegEx::operator = (const RegEx& rhs)
-	{
-		delete m_pOp;
-		m_pOp = 0;
-
-		m_op = rhs.m_op;
-		m_a = rhs.m_a;
-		m_z = rhs.m_z;
-		m_params = rhs.m_params;
-
-		SetOp();
-
-		return *this;
-	}
-
-	void RegEx::SetOp()
-	{
-		delete m_pOp;
-		m_pOp = 0;
-		switch(m_op) {
-			case REGEX_MATCH: m_pOp = new MatchOperator; break;
-			case REGEX_RANGE: m_pOp = new RangeOperator; break;
-			case REGEX_OR: m_pOp = new OrOperator; break;
-			case REGEX_AND: m_pOp = new AndOperator; break;
-			case REGEX_NOT: m_pOp = new NotOperator; break;
-			case REGEX_SEQ: m_pOp = new SeqOperator; break;
-			default: break;
-		}
-	}
-
-	bool RegEx::Matches(char ch) const
-	{
-		std::string str;
-		str += ch;
-		return Matches(str);
-	}
-
-	bool RegEx::Matches(const std::string& str) const
-	{
-		return Match(str) >= 0;
-	}
-
-	bool RegEx::Matches(const Buffer& buffer) const
-	{
-		return Match(buffer) >= 0;
 	}
 	
-	bool RegEx::Matches(const Stream& in) const
-	{
-		return Match(in) >= 0;
-	}
-
-	// Match
-	// . Matches the given string against this regular expression.
-	// . Returns the number of characters matched.
-	// . Returns -1 if no characters were matched (the reason for
-	//   not returning zero is that we may have an empty regex
-	//   which is ALWAYS successful at matching zero characters).
-	// . REMEMBER that we only match from the start of the buffer!
-	int RegEx::Match(const Buffer& buffer) const
-	{
-		if(!m_pOp)
-			return !buffer ? 0 : -1;  // the empty regex only is successful on the empty string
-
-		return m_pOp->Match(buffer, *this);
-	}
-
-	int RegEx::Match(const std::string& str) const
-	{
-		Buffer buffer(str.c_str(), str.size());
-		return Match(buffer);
-	}
-
-	// Match
-	int RegEx::Match(const Stream& in) const
-	{
-		return Match(in.current());
-	}
-
+	// combination constructors
 	RegEx operator ! (const RegEx& ex)
 	{
 		RegEx ret(REGEX_NOT);
 		ret.m_params.push_back(ex);
 		return ret;
 	}
-
+	
 	RegEx operator || (const RegEx& ex1, const RegEx& ex2)
 	{
 		RegEx ret(REGEX_OR);
@@ -141,7 +41,7 @@ namespace YAML
 		ret.m_params.push_back(ex2);
 		return ret;
 	}
-
+	
 	RegEx operator && (const RegEx& ex1, const RegEx& ex2)
 	{
 		RegEx ret(REGEX_AND);
@@ -149,85 +49,13 @@ namespace YAML
 		ret.m_params.push_back(ex2);
 		return ret;
 	}
-
+	
 	RegEx operator + (const RegEx& ex1, const RegEx& ex2)
 	{
 		RegEx ret(REGEX_SEQ);
 		ret.m_params.push_back(ex1);
 		ret.m_params.push_back(ex2);
 		return ret;
-	}
-
-	//////////////////////////////////////////////////////////////////////////////
-	// Operators
-
-	// MatchOperator
-	int RegEx::MatchOperator::Match(const Buffer& buffer, const RegEx& regex) const
-	{
-		if(!buffer || buffer[0] != regex.m_a)
-			return -1;
-		return 1;
-	}
-
-	// RangeOperator
-	int RegEx::RangeOperator::Match(const Buffer& buffer, const RegEx& regex) const
-	{
-		if(!buffer || regex.m_a > buffer[0] || regex.m_z < buffer[0])
-			return -1;
-		return 1;
-	}
-
-	// OrOperator
-	int RegEx::OrOperator::Match(const Buffer& buffer, const RegEx& regex) const
-	{
-		for(unsigned i=0;i<regex.m_params.size();i++) {
-			int n = regex.m_params[i].Match(buffer);
-			if(n >= 0)
-				return n;
-		}
-		return -1;
-	}
-
-	// AndOperator
-	// Note: 'AND' is a little funny, since we may be required to match things
-	//       of different lengths. If we find a match, we return the length of
-	//       the FIRST entry on the list.
-	int RegEx::AndOperator::Match(const Buffer& buffer, const RegEx& regex) const
-	{
-		int first = -1;
-		for(unsigned i=0;i<regex.m_params.size();i++) {
-			int n = regex.m_params[i].Match(buffer);
-			if(n == -1)
-				return -1;
-			if(i == 0)
-				first = n;
-		}
-		return first;
-	}
-
-	// NotOperator
-	int RegEx::NotOperator::Match(const Buffer& buffer, const RegEx& regex) const
-	{
-		if(regex.m_params.empty())
-			return -1;
-		if(regex.m_params[0].Match(buffer) >= 0)
-			return -1;
-		return 1;
-	}
-
-	// SeqOperator
-	int RegEx::SeqOperator::Match(const Buffer& buffer, const RegEx& regex) const
-	{
-		int offset = 0;
-		for(unsigned i=0;i<regex.m_params.size();i++) {
-			int n = regex.m_params[i].Match(buffer + offset);
-			if(n == -1)
-				return -1;
-
-			offset += n;
-		}
-
-		return offset;
-	}
+	}	
 }
 

src/regex.h

@@ -6,7 +6,6 @@
 namespace YAML
 {
 	class Stream;
-	struct Buffer;
 
 	enum REGEX_OP { REGEX_EMPTY, REGEX_MATCH, REGEX_RANGE, REGEX_OR, REGEX_AND, REGEX_NOT, REGEX_SEQ };
 
@@ -15,70 +14,46 @@ namespace YAML
 	// . Only matches from start of string
 	class RegEx
 	{
-	private:
-		// the operators
-		struct Operator {
-			virtual ~Operator() {}
-			virtual int Match(const Buffer& buffer, const RegEx& regex) const = 0;
-		};
-
-		struct MatchOperator: public Operator {
-			virtual int Match(const Buffer& buffer, const RegEx& regex) const;
-		};
-
-		struct RangeOperator: public Operator {
-			virtual int Match(const Buffer& buffer, const RegEx& regex) const;
-		};
-
-		struct OrOperator: public Operator {
-			virtual int Match(const Buffer& buffer, const RegEx& regex) const;
-		};
-
-		struct AndOperator: public Operator {
-			virtual int Match(const Buffer& buffer, const RegEx& regex) const;
-		};
-
-		struct NotOperator: public Operator {
-			virtual int Match(const Buffer& buffer, const RegEx& regex) const;
-		};
-
-		struct SeqOperator: public Operator {
-			virtual int Match(const Buffer& buffer, const RegEx& regex) const;
-		};
-
 	public:
-		friend struct Operator;
-
 		RegEx();
 		RegEx(char ch);
 		RegEx(char a, char z);
 		RegEx(const std::string& str, REGEX_OP op = REGEX_SEQ);
-		RegEx(const RegEx& rhs);
-		~RegEx();
-
-		RegEx& operator = (const RegEx& rhs);
+		~RegEx() {}
 
+		friend RegEx operator ! (const RegEx& ex);
+		friend RegEx operator || (const RegEx& ex1, const RegEx& ex2);
+		friend RegEx operator && (const RegEx& ex1, const RegEx& ex2);
+		friend RegEx operator + (const RegEx& ex1, const RegEx& ex2);
+		
 		bool Matches(char ch) const;
 		bool Matches(const std::string& str) const;
-		bool Matches(const Buffer& buffer) const;
 		bool Matches(const Stream& in) const;
+		template <typename Source> bool Matches(const Source& source) const;
+
 		int Match(const std::string& str) const;
-		int Match(const Buffer& buffer) const;
 		int Match(const Stream& in) const;
 
-		friend RegEx operator ! (const RegEx& ex);
-		friend RegEx operator || (const RegEx& ex1, const RegEx& ex2);
-		friend RegEx operator && (const RegEx& ex1, const RegEx& ex2);
-		friend RegEx operator + (const RegEx& ex1, const RegEx& ex2);
-
 	private:
 		RegEx(REGEX_OP op);
-		void SetOp();
+		
+		template <typename Source> bool IsValidSource(const Source& source) const;
+		template <typename Source> int Match(const Source& source) const;
+		template <typename Source> int MatchUnchecked(const Source& source) const;
+
+		template <typename Source> int MatchOpEmpty(const Source& source) const;
+		template <typename Source> int MatchOpMatch(const Source& source) const;
+		template <typename Source> int MatchOpRange(const Source& source) const;
+		template <typename Source> int MatchOpOr(const Source& source) const;
+		template <typename Source> int MatchOpAnd(const Source& source) const;
+		template <typename Source> int MatchOpNot(const Source& source) const;
+		template <typename Source> int MatchOpSeq(const Source& source) const;
 
 	private:
 		REGEX_OP m_op;
-		Operator *m_pOp;
 		char m_a, m_z;
 		std::vector <RegEx> m_params;
 	};
 }
+
+#include "regeximpl.h"

src/regeximpl.h

+#pragma once
+
+#include "stream.h"
+#include "stringsource.h"
+#include "streamcharsource.h"
+
+namespace YAML
+{
+	// query matches
+	inline bool RegEx::Matches(char ch) const {
+		std::string str;
+		str += ch;
+		return Matches(str);
+	}
+	
+	inline bool RegEx::Matches(const std::string& str) const {
+		return Match(str) >= 0;
+	}
+	
+	inline bool RegEx::Matches(const Stream& in) const {
+		return Match(in) >= 0;
+	}
+
+	template <typename Source>
+	inline bool RegEx::Matches(const Source& source) const {
+		return Match(source) >= 0;
+	}
+
+	// Match
+	// . Matches the given string against this regular expression.
+	// . Returns the number of characters matched.
+	// . Returns -1 if no characters were matched (the reason for
+	//   not returning zero is that we may have an empty regex
+	//   which is ALWAYS successful at matching zero characters).
+	// . REMEMBER that we only match from the start of the buffer!	
+	inline int RegEx::Match(const std::string& str) const
+	{
+		StringCharSource source(str.c_str(), str.size());
+		return Match(source);
+	}
+	
+	inline int RegEx::Match(const Stream& in) const
+	{
+		StreamCharSource source(in);
+		return Match(source);
+	}
+
+	template <typename Source>
+	inline bool RegEx::IsValidSource(const Source& source) const
+	{
+		return source;
+	}
+	
+	template<>
+	inline bool RegEx::IsValidSource<StringCharSource>(const StringCharSource&source) const
+	{
+		return source || m_op == REGEX_EMPTY;
+	}
+
+	template <typename Source>
+	inline int RegEx::Match(const Source& source) const
+	{
+		return IsValidSource(source) ? MatchUnchecked(source) : -1;
+	}
+	
+	template <typename Source>
+	inline int RegEx::MatchUnchecked(const Source& source) const
+	{
+		switch(m_op) {
+			case REGEX_EMPTY:
+				return MatchOpEmpty(source);
+			case REGEX_MATCH:
+				return MatchOpMatch(source);
+			case REGEX_RANGE:
+				return MatchOpRange(source);
+			case REGEX_OR:
+				return MatchOpOr(source);
+			case REGEX_AND:
+				return MatchOpAnd(source);
+			case REGEX_NOT:
+				return MatchOpNot(source);
+			case REGEX_SEQ:
+				return MatchOpSeq(source);
+		}
+		
+		return -1;
+	}
+
+	//////////////////////////////////////////////////////////////////////////////
+	// Operators
+	// Note: the convention MatchOp*<Source> is that we can assume IsSourceValid(source).
+	//       So we do all our checks *before* we call these functions
+	
+	// EmptyOperator
+	template <typename Source>
+	inline int RegEx::MatchOpEmpty(const Source& source) const {
+		return source[0] == Stream::eof() ? 0 : -1;
+	}
+	
+	template <>
+	inline int RegEx::MatchOpEmpty<StringCharSource>(const StringCharSource& source) const {
+		return !source ? 0 : -1;  // the empty regex only is successful on the empty string
+	}
+
+	// MatchOperator
+	template <typename Source>
+	inline int RegEx::MatchOpMatch(const Source& source) const {
+		if(source[0] != m_a)
+			return -1;
+		return 1;
+	}
+	
+	// RangeOperator
+	template <typename Source>
+	inline int RegEx::MatchOpRange(const Source& source) const {
+		if(m_a > source[0] || m_z < source[0])
+			return -1;
+		return 1;
+	}
+	
+	// OrOperator
+	template <typename Source>
+	inline int RegEx::MatchOpOr(const Source& source) const {
+		for(unsigned i=0;i<m_params.size();i++) {
+			int n = m_params[i].MatchUnchecked(source);
+			if(n >= 0)
+				return n;
+		}
+		return -1;
+	}
+	
+	// AndOperator
+	// Note: 'AND' is a little funny, since we may be required to match things
+	//       of different lengths. If we find a match, we return the length of
+	//       the FIRST entry on the list.
+	template <typename Source>
+	inline int RegEx::MatchOpAnd(const Source& source) const {
+		int first = -1;
+		for(unsigned i=0;i<m_params.size();i++) {
+			int n = m_params[i].MatchUnchecked(source);
+			if(n == -1)
+				return -1;
+			if(i == 0)
+				first = n;
+		}
+		return first;
+	}
+	
+	// NotOperator
+	template <typename Source>
+	inline int RegEx::MatchOpNot(const Source& source) const {
+		if(m_params.empty())
+			return -1;
+		if(m_params[0].MatchUnchecked(source) >= 0)
+			return -1;
+		return 1;
+	}
+	
+	// SeqOperator
+	template <typename Source>
+	inline int RegEx::MatchOpSeq(const Source& source) const {
+		int offset = 0;
+		for(unsigned i=0;i<m_params.size();i++) {
+			int n = m_params[i].Match(source + offset); // note Match, not MatchUnchecked because we need to check validity after the offset
+			if(n == -1)
+				return -1;
+			offset += n;
+		}
+		
+		return offset;
+	}
+}

src/stream.h

 #pragma once
 
+#include <deque>
 #include <ios>
 #include <string>
+#include <iostream>
+#include <set>
 
 namespace YAML
 {
-	// a simple buffer wrapper that knows how big it is
-	struct Buffer {
-		Buffer(const char *b, int s): buffer(b), size(s) {}
-
-		operator bool() const { return size > 0; }
-		bool operator !() const { return !static_cast <bool> (*this); }
-		char operator [] (int i) const { return buffer[i]; }
-		const Buffer operator + (int offset) const { return Buffer(buffer + offset, size - offset); }
-		Buffer& operator ++ () { ++buffer; --size; return *this; }
-
-		const char *buffer;
-		int size;
-	};
+	static const size_t MAX_PARSER_PUSHBACK = 8;
 
 	class Stream
 	{
 	public:
+		friend class StreamCharSource;
+		
 		Stream(std::istream& input);
 		~Stream();
 
 		operator bool() const;
 		bool operator !() const { return !static_cast <bool>(*this); }
 
-		const Buffer current() const { return Buffer(buffer + pos, size - pos); }
-		char peek();
+		char peek() const;
 		char get();
 		std::string get(int n);
 		void eat(int n = 1);
 
-		int pos, line, column, size;
+		static char eof() { return 0x04; }
+
+		int pos, line, column;
 	
 	private:
-		char *buffer;
+		enum CharacterSet {utf8, utf16le, utf16be, utf32le, utf32be};
+
+		std::istream& m_input;
+		CharacterSet m_charSet;
+		unsigned char m_bufPushback[MAX_PARSER_PUSHBACK];
+		mutable size_t m_nPushedBack;
+		mutable std::deque<char> m_readahead;
+		unsigned char* const m_pPrefetched;
+		mutable size_t m_nPrefetchedAvailable;
+		mutable size_t m_nPrefetchedUsed;
+		
+		void AdvanceCurrent();
+		char CharAt(size_t i) const;
+		bool ReadAheadTo(size_t i) const;
+		bool _ReadAheadTo(size_t i) const;
+		void StreamInUtf8() const;
+		void StreamInUtf16() const;
+		void StreamInUtf32() const;
+		unsigned char GetNextByte() const;
 	};
+
+	// CharAt
+	// . Unchecked access
+	inline char Stream::CharAt(size_t i) const {
+		return m_readahead[i];
+	}
+	
+	inline bool Stream::ReadAheadTo(size_t i) const {
+		if(m_readahead.size() > i)
+			return true;
+		return _ReadAheadTo(i);
+	}	
 }

src/streamcharsource.h

+#pragma once
+
+#include <cstddef>
+
+namespace YAML
+{
+	class StreamCharSource
+	{
+	public:
+		StreamCharSource(const Stream& stream);
+		~StreamCharSource() {}
+			
+		operator bool() const;
+		char operator [] (std::size_t i) const { return m_stream.CharAt(m_offset + i); }
+		bool operator !() const { return !static_cast<bool>(*this); }
+
+		const StreamCharSource operator + (int i) const;
+			
+	private:
+		std::size_t m_offset;
+		const Stream& m_stream;
+	};
+	
+	inline StreamCharSource::StreamCharSource(const Stream& stream): m_offset(0), m_stream(stream) {
+	}
+	
+	inline StreamCharSource::operator bool() const {
+		return m_stream.ReadAheadTo(m_offset);
+	}
+	
+	inline const StreamCharSource StreamCharSource::operator + (int i) const {
+		StreamCharSource source(*this);
+		if(static_cast<int> (source.m_offset) + i >= 0)
+			source.m_offset += i;
+		else
+			source.m_offset = 0;
+		return source;
+	}
+}

src/stringsource.h

+#pragma once
+
+#include <cstddef>
+
+namespace YAML
+{
+	class StringCharSource
+	{
+	public:
+		StringCharSource(const char *str, std::size_t size): m_str(str), m_size(size), m_offset(0) {}
+
+		operator bool() const { return m_offset < m_size; }
+		char operator [] (std::size_t i) const { return m_str[m_offset + i]; }
+		bool operator !() const { return !static_cast<bool>(*this); }
+
+		const StringCharSource operator + (int i) const {
+			StringCharSource source(*this);
+			if(static_cast<int> (source.m_offset) + i >= 0)
+				source.m_offset += i;
+			else
+				source.m_offset = 0;
+			return source;
+		}
+			
+		StringCharSource& operator ++ () {
+			++m_offset;
+			return *this;
+		}
+	private:
+		const char *m_str;
+		std::size_t m_size;
+		std::size_t m_offset;
+	};
+}

util/CMakeLists.txt

+add_executable(parse parse.cpp)
+target_link_libraries(parse yaml-cpp)

util/parse.cpp

+#include "yaml.h"
+#include <fstream>
+#include <iostream>
+
+int main(int argc, char **argv)
+{
+	if(argc != 2) {
+		std::cout << "Usage: " << argv[0] << " input-file\n";
+		return 0;
+	}
+
+	std::ifstream fin(argv[1]);
+	try {
+		YAML::Parser parser(fin);
+		YAML::Node doc;
+		parser.GetNextDocument(doc);
+	} catch(const YAML::Exception& e) {
+		std::cerr << "Error at line " << e.line << ", col " << e.column << ": " << e.msg << "\n";
+	}
+	return 0;
+}

yaml-reader/tests.cpp

@@ -72,6 +72,174 @@ namespace Test
 					std::cout << "Caught exception: " << error << "\n";
 			}
 		}
+
+		typedef void (*EncodingFn)(std::ostream&, int);
+
+		inline char Byte(int ch)
+		{
+			return static_cast<char>(static_cast<unsigned char>(static_cast<unsigned int>(ch)));
+		}
+
+		void EncodeToUtf8(std::ostream& stream, int ch)
+		{
+			if (ch <= 0x7F)
+			{
+				stream << Byte(ch);
+			}
+			else if (ch <= 0x7FF)
+			{
+				stream << Byte(0xC0 | (ch >> 6));
+				stream << Byte(0x80 | (ch & 0x3F));
+			}
+			else if (ch <= 0xFFFF)
+			{
+				stream << Byte(0xE0 | (ch >> 12));
+				stream << Byte(0x80 | ((ch >> 6) & 0x3F));
+				stream << Byte(0x80 | (ch & 0x3F));
+			}
+			else if (ch <= 0x1FFFFF)
+			{
+				stream << Byte(0xF0 | (ch >> 18));
+				stream << Byte(0x80 | ((ch >> 12) & 0x3F));
+				stream << Byte(0x80 | ((ch >> 6) & 0x3F));
+				stream << Byte(0x80 | (ch & 0x3F));
+			}
+		}
+
+		bool SplitUtf16HighChar(std::ostream& stream, EncodingFn encoding, int ch)
+		{
+			int biasedValue = ch - 0x10000;
+			if (biasedValue < 0)
+			{
+				return false;
+			}
+			int high = 0xD800 | (biasedValue >> 10);
+			int low  = 0xDC00 | (biasedValue & 0x3FF);
+			encoding(stream, high);
+			encoding(stream, low);
+			return true;
+		}
+
+		void EncodeToUtf16LE(std::ostream& stream, int ch)
+		{
+			if (!SplitUtf16HighChar(stream, &EncodeToUtf16LE, ch))
+			{
+				stream << Byte(ch & 0xFF) << Byte(ch >> 8);
+			}
+		}
+
+		void EncodeToUtf16BE(std::ostream& stream, int ch)
+		{
+			if (!SplitUtf16HighChar(stream, &EncodeToUtf16BE, ch))
+			{
+				stream << Byte(ch >> 8) << Byte(ch & 0xFF);
+			}
+		}
+
+		void EncodeToUtf32LE(std::ostream& stream, int ch)
+		{
+			stream << Byte(ch & 0xFF) << Byte((ch >> 8) & 0xFF) 
+				<< Byte((ch >> 16) & 0xFF) << Byte((ch >> 24) & 0xFF);
+		}
+
+		void EncodeToUtf32BE(std::ostream& stream, int ch)
+		{
+			stream << Byte((ch >> 24) & 0xFF) << Byte((ch >> 16) & 0xFF)
+				<< Byte((ch >> 8) & 0xFF) << Byte(ch & 0xFF);
+		}
+
+		class EncodingTester
+		{
+		public:
+			EncodingTester(EncodingFn encoding, bool declareEncoding)
+			{
+				if (declareEncoding)
+				{
+					encoding(m_yaml, 0xFEFF);
+				}
+
+				AddEntry(encoding, 0x0021, 0x007E); // Basic Latin
+				AddEntry(encoding, 0x00A1, 0x00FF); // Latin-1 Supplement
+				AddEntry(encoding, 0x0660, 0x06FF); // Arabic (largest contiguous block)
+
+				// CJK unified ideographs (multiple lines)
+				AddEntry(encoding, 0x4E00, 0x4EFF);
+				AddEntry(encoding, 0x4F00, 0x4FFF);
+				AddEntry(encoding, 0x5000, 0x51FF); // 512 character line
+				AddEntry(encoding, 0x5200, 0x54FF); // 768 character line
+				AddEntry(encoding, 0x5500, 0x58FF); // 1024 character line
+
+				AddEntry(encoding, 0x103A0, 0x103C3); // Old Persian
+
+				m_yaml.seekg(0, std::ios::beg);
+			}
+
+			std::istream& stream() {return m_yaml;}
+			const std::vector<std::string>& entries() {return m_entries;}
+
+		private:
+			std::stringstream m_yaml;
+			std::vector<std::string> m_entries;
+
+			void AddEntry(EncodingFn encoding, int startCh, int endCh)
+			{
+				encoding(m_yaml, '-');
+				encoding(m_yaml, ' ');
+				encoding(m_yaml, '|');
+				encoding(m_yaml, '\n');
+				encoding(m_yaml, ' ');
+				encoding(m_yaml, ' ');
+
+				std::stringstream entry;
+				for (int ch = startCh; ch <= endCh; ++ch)
+				{
+					encoding(m_yaml, ch);
+					EncodeToUtf8(entry, ch);
+				}
+				encoding(m_yaml, '\n');
+
+				m_entries.push_back(entry.str());
+			}
+		};
+
+		void RunEncodingTest(EncodingFn encoding, bool declareEncoding, const std::string& name, bool& passed)
+		{
+			EncodingTester tester(encoding, declareEncoding);
+			std::string error;
+			bool ok = true;
+			try {
+				YAML::Parser parser(tester.stream());
+				YAML::Node doc;
+				parser.GetNextDocument(doc);
+
+				YAML::Iterator itNode = doc.begin();
+				std::vector<std::string>::const_iterator itEntry = tester.entries().begin();
+				for (; (itNode != doc.end()) && (itEntry != tester.entries().end()); ++itNode, ++itEntry)
+				{
+					std::string stScalarValue;
+					if (!itNode->GetScalar(stScalarValue) && (stScalarValue == *itEntry))
+					{
+						break;
+					}
+				}
+
+				if ((itNode != doc.end()) || (itEntry != tester.entries().end()))
+				{
+					ok = false;
+				}
+			} catch(const YAML::Exception& e) {
+				ok = false;
+				error = e.msg;
+			}
+			if(ok) {
+				std::cout << "Parser test passed: " << name << "\n";
+			} else {
+				passed = false;
+				std::cout << "Parser test failed: " << name << "\n";
+				if(error != "")
+					std::cout << "Caught exception: " << error << "\n";
+			}
+		}
 	}
 
 	bool RunParserTests()
@@ -94,6 +262,17 @@ namespace Test
 		RunParserTest(&Parser::SimpleMap, "simple map", passed);
 		RunParserTest(&Parser::FlowSeq, "flow seq", passed);
 		RunParserTest(&Parser::FlowMap, "flow map", passed);
+
+		RunEncodingTest(&EncodeToUtf8, false, "UTF-8, no BOM", passed);
+		RunEncodingTest(&EncodeToUtf8, true, "UTF-8 with BOM", passed);
+		RunEncodingTest(&EncodeToUtf16LE, false, "UTF-16LE, no BOM", passed);
+		RunEncodingTest(&EncodeToUtf16LE, true, "UTF-16LE with BOM", passed);
+		RunEncodingTest(&EncodeToUtf16BE, false, "UTF-16BE, no BOM", passed);
+		RunEncodingTest(&EncodeToUtf16BE, true, "UTF-16BE with BOM", passed);
+		RunEncodingTest(&EncodeToUtf32LE, false, "UTF-32LE, no BOM", passed);
+		RunEncodingTest(&EncodeToUtf32LE, true, "UTF-32LE with BOM", passed);
+		RunEncodingTest(&EncodeToUtf32BE, false, "UTF-32BE, no BOM", passed);
+		RunEncodingTest(&EncodeToUtf32BE, true, "UTF-32BE with BOM", passed);
 		return passed;
 	}
 

yamlcpp.vcproj

@@ -231,6 +231,10 @@
 					RelativePath=".\src\stream.cpp"
 					>
 				</File>
+				<File
+					RelativePath=".\src\stringsource.cpp"
+					>
+				</File>
 			</Filter>
 			<Filter
 				Name="Emitter"
@@ -358,6 +362,10 @@
 					>
 				</File>
 				<File
+					RelativePath=".\src\stringsource.h"
+					>
+				</File>
+				<File
 					RelativePath=".\src\token.h"
 					>
 				</File>