Added simple keys.

There's a bug (and question): should we test simple keys' validity BEFORE stuff or AFTER stuff?

exceptions.h

@@ -14,6 +14,8 @@ namespace YAML
 	class IllegalTabInScalar: public Exception {};
 	class DocIndicatorInQuote: public Exception {};
 	class EOFInQuote: public Exception {};
+	class RequiredSimpleKeyNotFound: public Exception {};
+
 	class UnknownEscapeSequence: public Exception {
 	public:
 		UnknownEscapeSequence(char ch_): ch(ch_) {}

exp.cpp

@@ -46,18 +46,18 @@ namespace YAML
 			unsigned value = ParseHex(str);
 
 			// legal unicode?
-			if((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF)
-				throw InvalidUnicode(value);
-
-			// now break it up into chars
-			if(value <= 0x7F)
-				return Str(value);
-			else if(value <= 0x7FF)
-				return Str(0xC0 + (value >> 6)) + Str(0x80 + (value & 0x3F));
-			else if(value <= 0xFFFF)
-				return Str(0xE0 + (value >> 12)) + Str(0x80 + ((value >> 6) & 0x3F)) + Str(0x80 + (value & 0x3F));
-			else
-				return Str(0xF0 + (value >> 18)) + Str(0x80 + ((value >> 12) & 0x3F)) +
+			if((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF)
+				throw InvalidUnicode(value);
+
+			// now break it up into chars
+			if(value <= 0x7F)
+				return Str(value);
+			else if(value <= 0x7FF)
+				return Str(0xC0 + (value >> 6)) + Str(0x80 + (value & 0x3F));
+			else if(value <= 0xFFFF)
+				return Str(0xE0 + (value >> 12)) + Str(0x80 + ((value >> 6) & 0x3F)) + Str(0x80 + (value & 0x3F));
+			else
+				return Str(0xF0 + (value >> 18)) + Str(0x80 + ((value >> 12) & 0x3F)) +
 					Str(0x80 + ((value >> 6) & 0x3F)) + Str(0x80 + (value & 0x3F));
 		}
 
@@ -76,27 +76,27 @@ namespace YAML
 
 			// switch on escape character
 			char ch = in.get();
-			switch(ch) {
-				case '0': return "\0";
-				case 'a': return "\x07";
-				case 'b': return "\x08";
-				case 't':
-                case '\t': return "\x09";
-				case 'n': return "\x0A";
-				case 'v': return "\x0B";
-				case 'f': return "\x0C";
-				case 'r': return "\x0D";
-				case 'e': return "\x1B";
-				case ' ': return "\x20";
-				case '\"': return "\"";
-				case '\'': return "\'";
-				case '\\': return "\\";
-				case 'N': return "\xC2\x85";  // NEL (#x85)
-				case '_': return "\xC2\xA0";  // #xA0
-				case 'L': return "\xE2\x80\xA8";  // LS (#x2028)
-				case 'P': return "\xE2\x80\xA9";  // PS (#x2029)
-				case 'x': return Escape(in, length, 2);
-				case 'u': return Escape(in, length, 4);
+			switch(ch) {
+				case '0': return "\0";
+				case 'a': return "\x07";
+				case 'b': return "\x08";
+				case 't':
+                case '\t': return "\x09";
+				case 'n': return "\x0A";
+				case 'v': return "\x0B";
+				case 'f': return "\x0C";
+				case 'r': return "\x0D";
+				case 'e': return "\x1B";
+				case ' ': return "\x20";
+				case '\"': return "\"";
+				case '\'': return "\'";
+				case '\\': return "\\";
+				case 'N': return "\xC2\x85";  // NEL (#x85)
+				case '_': return "\xC2\xA0";  // #xA0
+				case 'L': return "\xE2\x80\xA8";  // LS (#x2028)
+				case 'P': return "\xE2\x80\xA9";  // PS (#x2029)
+				case 'x': return Escape(in, length, 2);
+				case 'u': return Escape(in, length, 4);
 				case 'U': return Escape(in, length, 8);
 			}
 

scanner.cpp

@@ -7,7 +7,8 @@
 namespace YAML
 {
 	Scanner::Scanner(std::istream& in)
-		: INPUT(in), m_startedStream(false), m_endedStream(false), m_simpleKeyAllowed(false), m_flowLevel(0), m_column(0)
+		: INPUT(in), m_startedStream(false), m_endedStream(false), m_simpleKeyAllowed(false), m_flowLevel(0),
+		m_line(0), m_column(0)
 	{
 	}
 
@@ -32,8 +33,10 @@ namespace YAML
 	{
 		m_column++;
 		char ch = INPUT.get();
-		if(ch == '\n')
+		if(ch == '\n') {
 			m_column = 0;
+			m_line++;
+		}
 		return ch;
 	}
 
@@ -51,12 +54,8 @@ namespace YAML
 	// . Eats 'n' characters and updates our position.
 	void Scanner::Eat(int n)
 	{
-		for(int i=0;i<n;i++) {
-			m_column++;
-			char ch = INPUT.get();
-			if(ch == '\n')
-				m_column = 0;
-		}
+		for(int i=0;i<n;i++)
+			GetChar();
 	}
 
 	// GetLineBreak
@@ -147,6 +146,17 @@ namespace YAML
 		m_limboTokens.insert(pToken);
 		m_tokens.push(ScanToken(pToken));
 		m_limboTokens.erase(pToken);
+
+		// then remove impossible tokens
+		std::queue <Token *> temp;
+		while(!m_tokens.empty()) {
+			Token *pToken = m_tokens.front();
+			m_tokens.pop();
+			if(pToken->isPossible)
+				temp.push(pToken);
+		}
+
+		m_tokens = temp;
 	}
 
 	///////////////////////////////////////////////////////////////////////
@@ -243,6 +253,9 @@ namespace YAML
 			// otherwise, let's eat the line break and keep going
 			EatLineBreak();
 
+			// oh yeah, and let's get rid of that simple key
+			ValidateSimpleKey();
+
 			// new line - we may be able to accept a simple key now
 			if(m_flowLevel == 0)
 				m_simpleKeyAllowed = true;
@@ -252,15 +265,16 @@ namespace YAML
 	// PushIndentTo
 	// . Pushes an indentation onto the stack, and enqueues the
 	//   proper token (sequence start or mapping start).
-	void Scanner::PushIndentTo(int column, bool sequence)
+	// . Returns the token it generates (if any).
+	Token *Scanner::PushIndentTo(int column, bool sequence)
 	{
 		// are we in flow?
 		if(m_flowLevel > 0)
-			return;
+			return 0;
 
 		// is this actually an indentation?
 		if(column <= m_indents.top())
-			return;
+			return 0;
 
 		// now push
 		m_indents.push(column);
@@ -268,6 +282,8 @@ namespace YAML
 			m_tokens.push(new BlockSeqStartToken);
 		else
 			m_tokens.push(new BlockMapStartToken);
+
+		return m_tokens.front();
 	}
 
 	// PopIndentTo
@@ -312,6 +328,9 @@ namespace YAML
 
 			while(!m_tokens.empty()) {
 				Token *pToken = m_tokens.front();
+				if(!pToken->isValid)   // gotta wait on the invalid tokens - they might become valid!
+					break;
+
 				m_tokens.pop();
 				std::cout << typeid(*pToken).name() << ": " << *pToken << std::endl;
 				delete pToken;

scanner.h

@@ -18,11 +18,15 @@ namespace YAML
 
 		void ScanNextToken();
 		void ScanToNextToken();
-		void PushIndentTo(int column, bool sequence);
+		Token *PushIndentTo(int column, bool sequence);
 		void PopIndentTo(int column);
 		void IncreaseFlowLevel();
 		void DecreaseFlowLevel();
 
+		void InsertSimpleKey();
+		bool ValidateSimpleKey();
+		void ValidateAllSimpleKeys();
+
 		void Scan();
 
 	private:
@@ -41,6 +45,7 @@ namespace YAML
 
 		struct WhitespaceInfo {
 			WhitespaceInfo();
+
 			void AddBlank(char ch);
 			void AddBreak(const std::string& line);
 			std::string Join();
@@ -49,13 +54,24 @@ namespace YAML
 			std::string whitespace, leadingBreaks, trailingBreaks;
 		};
 
+		struct SimpleKey {
+			SimpleKey(int pos_, int line_, int column_);
+
+			void Validate();
+			void Invalidate();
+
+			int pos, line, column;
+			bool required;
+			Token *pMapStart, *pKey;
+		};
+
 		template <typename T> void ScanAndEnqueue(T *pToken);
 		template <typename T> T *ScanToken(T *pToken);
 
 	private:
 		// the stream
 		std::istream& INPUT;
-		int m_column;
+		int m_line, m_column;
 
 		// the output (tokens)
 		std::queue <Token *> m_tokens;
@@ -65,6 +81,7 @@ namespace YAML
 		bool m_startedStream, m_endedStream;
 		bool m_simpleKeyAllowed;
 		int m_flowLevel;                // number of unclosed '[' and '{' indicators
+		std::stack <SimpleKey> m_simpleKeys;
 		std::stack <int> m_indents;
 	};
 }

scantoken.cpp

@@ -26,7 +26,7 @@ namespace YAML
 			m_column = 0;
 
 		PopIndentTo(-1);
-		// TODO: "reset simple keys"
+		ValidateAllSimpleKeys();
 
 		m_simpleKeyAllowed = false;
 		m_endedStream = true;
@@ -38,8 +38,7 @@ namespace YAML
 	template <> DocumentStartToken *Scanner::ScanToken(DocumentStartToken *pToken)
 	{
 		PopIndentTo(m_column);
-		// TODO: "reset simple keys"
-
+		ValidateAllSimpleKeys();
 		m_simpleKeyAllowed = false;
 
 		// eat
@@ -51,8 +50,7 @@ namespace YAML
 	template <> DocumentEndToken *Scanner::ScanToken(DocumentEndToken *pToken)
 	{
 		PopIndentTo(-1);
-		// TODO: "reset simple keys"
-
+		ValidateAllSimpleKeys();
 		m_simpleKeyAllowed = false;
 
 		// eat
@@ -63,8 +61,8 @@ namespace YAML
 	// FlowSeqStartToken
 	template <> FlowSeqStartToken *Scanner::ScanToken(FlowSeqStartToken *pToken)
 	{
-		// TODO: "save simple key"
-
+		// flow sequences can be simple keys
+		InsertSimpleKey();
 		IncreaseFlowLevel();
 		m_simpleKeyAllowed = true;
 
@@ -76,8 +74,8 @@ namespace YAML
 	// FlowMapStartToken
 	template <> FlowMapStartToken *Scanner::ScanToken(FlowMapStartToken *pToken)
 	{
-		// TODO: "save simple key"
-
+		// flow maps can be simple keys
+		InsertSimpleKey();
 		IncreaseFlowLevel();
 		m_simpleKeyAllowed = true;
 
@@ -89,8 +87,7 @@ namespace YAML
 	// FlowSeqEndToken
 	template <> FlowSeqEndToken *Scanner::ScanToken(FlowSeqEndToken *pToken)
 	{
-		// TODO: "remove simple key"
-
+//		ValidateSimpleKey();
 		DecreaseFlowLevel();
 		m_simpleKeyAllowed = false;
 
@@ -102,8 +99,7 @@ namespace YAML
 	// FlowMapEndToken
 	template <> FlowMapEndToken *Scanner::ScanToken(FlowMapEndToken *pToken)
 	{
-		// TODO: "remove simple key"
-
+		ValidateSimpleKey();
 		DecreaseFlowLevel();
 		m_simpleKeyAllowed = false;
 
@@ -115,8 +111,7 @@ namespace YAML
 	// FlowEntryToken
 	template <> FlowEntryToken *Scanner::ScanToken(FlowEntryToken *pToken)
 	{
-		// TODO: "remove simple key"
-
+		ValidateSimpleKey();
 		m_simpleKeyAllowed = true;
 
 		// eat
@@ -127,6 +122,8 @@ namespace YAML
 	// BlockEntryToken
 	template <> BlockEntryToken *Scanner::ScanToken(BlockEntryToken *pToken)
 	{
+		ValidateSimpleKey();
+
 		// we better be in the block context!
 		if(m_flowLevel == 0) {
 			// can we put it here?
@@ -138,8 +135,6 @@ namespace YAML
 			// TODO: throw?
 		}
 
-		// TODO: "remove simple key"
-
 		m_simpleKeyAllowed = true;
 
 		// eat
@@ -174,10 +169,13 @@ namespace YAML
 	// ValueToken
 	template <> ValueToken *Scanner::ScanToken(ValueToken *pToken)
 	{
-		// TODO: Is it a simple key?
-		if(false) {
+		// does this follow a simple key?
+		bool isValidKey = ValidateSimpleKey();
+
+		if(isValidKey) {
+			// can't follow a simple key with another simple key (dunno why, though - it seems fine)
+			m_simpleKeyAllowed = false;
 		} else {
-			// If not, ...
 			// are we in block context?
 			if(m_flowLevel == 0) {
 				if(!m_simpleKeyAllowed)
@@ -185,13 +183,13 @@ namespace YAML
 
 				PushIndentTo(m_column, false);
 			}
-		}
 
-		// can only put a simple key here if we're in block context
-		if(m_flowLevel == 0)
-			m_simpleKeyAllowed = true;
-		else
-			m_simpleKeyAllowed = false;
+			// can only put a simple key here if we're in block context
+			if(m_flowLevel == 0)
+				m_simpleKeyAllowed = true;
+			else
+				m_simpleKeyAllowed = false;
+		}
 
 		// eat
 		Eat(1);
@@ -205,8 +203,8 @@ namespace YAML
 	//   and in-line whitespace (which is kept) separately.
 	template <> PlainScalarToken *Scanner::ScanToken(PlainScalarToken *pToken)
 	{
-		// TODO: "save simple key"
-
+		// insert a potential simple key
+		InsertSimpleKey();
 		m_simpleKeyAllowed = false;
 
 		// now eat and store the scalar
@@ -255,6 +253,9 @@ namespace YAML
 					int n = Exp::Break.Match(INPUT);
 					std::string line = GetChar(n);
 					info.AddBreak(line);
+
+					// and we can't continue a simple key to the next line
+					ValidateSimpleKey();
 				}
 			}
 
@@ -277,8 +278,8 @@ namespace YAML
 	// QuotedScalarToken
 	template <> QuotedScalarToken *Scanner::ScanToken(QuotedScalarToken *pToken)
 	{
-		// TODO: "save simple key"
-
+		// insert a potential simple key
+		InsertSimpleKey();
 		m_simpleKeyAllowed = false;
 
 		// eat single or double quote
@@ -341,6 +342,9 @@ namespace YAML
 					int n = Exp::Break.Match(INPUT);
 					std::string line = GetChar(n);
 					info.AddBreak(line);
+
+					// and we can't continue a simple key to the next line
+					ValidateSimpleKey();
 				}
 			}
 

simplekey.cpp

+#include "scanner.h"
+#include "token.h"
+#include "exceptions.h"
+#include "exp.h"
+
+namespace YAML
+{
+	Scanner::SimpleKey::SimpleKey(int pos_, int line_, int column_)
+		: pos(pos_), line(line_), column(column_), required(false), pMapStart(0), pKey(0)
+	{
+	}
+
+	void Scanner::SimpleKey::Validate()
+	{
+		if(pMapStart)
+			pMapStart->isValid = true;
+		if(pKey)
+			pKey->isValid = true;
+	}
+
+	void Scanner::SimpleKey::Invalidate()
+	{
+		if(required)
+			throw RequiredSimpleKeyNotFound();
+
+		if(pMapStart)
+			pMapStart->isPossible = false;
+		if(pKey)
+			pKey->isPossible = false;
+	}
+
+	// InsertSimpleKey
+	// . Adds a potential simple key to the queue,
+	//   and saves it on a stack.
+	void Scanner::InsertSimpleKey()
+	{
+		SimpleKey key(INPUT.tellg(), m_line, m_column);
+
+		// first add a map start, if necessary
+		key.pMapStart = PushIndentTo(m_column, false);
+		if(key.pMapStart)
+			key.pMapStart->isValid = false;
+//		else
+//			key.required = true;	// TODO: is this correct?
+
+		// then add the (now invalid) key
+		key.pKey = new KeyToken;
+		key.pKey->isValid = false;
+
+		m_tokens.push(key.pKey);
+
+		m_simpleKeys.push(key);
+	}
+
+	// ValidateSimpleKey
+	// . Determines whether the latest simple key to be added is valid,
+	//   and if so, makes it valid.
+	bool Scanner::ValidateSimpleKey()
+	{
+		if(m_simpleKeys.empty())
+			return false;
+
+		// grab top key
+		SimpleKey key = m_simpleKeys.top();
+		m_simpleKeys.pop();
+
+		bool isValid = true;
+
+		// needs to be followed immediately by a value
+		if(m_flowLevel > 0 && !Exp::ValueInFlow.Matches(INPUT))
+			isValid = false;
+		if(m_flowLevel == 0 && !Exp::Value.Matches(INPUT))
+			isValid = false;
+
+		// also needs to be less than 1024 characters and inline
+		if(m_line != key.line || (int) INPUT.tellg() - key.pos > 1024)
+			isValid = false;
+
+		// invalidate key
+		if(isValid)
+			key.Validate();
+		else
+			key.Invalidate();
+
+		// In block style, remember that we've pushed an indent for this potential simple key (if it was starting).
+		// If it was invalid, then we need to pop it off.
+		// Note: we're guaranteed to be popping the right one (i.e., there couldn't have been anything in
+		//       between) since keys have to be inline, and will be invalidated immediately on a newline.
+		if(!isValid && m_flowLevel == 0)
+			m_indents.pop();
+
+		return isValid;
+	}
+
+	void Scanner::ValidateAllSimpleKeys()
+	{
+		while(!m_simpleKeys.empty())
+			ValidateSimpleKey();
+	}
+}

test.yaml

 ---
-- milk and eggs
-- [cheddar, american, swiss]
-...
+model:
+    file: data/models/compound.model
+    textures: data/materials/compound
\ No newline at end of file
+rooms:
+    - name: "Room #1"
+      pos: [0, 0, 0]
+      size: [1000, 1000, 500]
+      height: 500
+      stairtype: none
+      display: []
+      pathfinding:
+          tilesize: 50
+          size: [24, 24]
+          map:
+              -----------------------
+              -+++++++++++++++++++++-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+---------------------
+              -+---------------------
+              -+---------------------
+              -+---------------------
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+++++++++++++++++++++-
+              -----------------------
+    - name: Doorway
+      pos: [1000, 400, 0]
+      size: [50, 200, 500]
+      height: 500
+      stairtype: none
+      display: []
+      pathfinding:
+          tilesize: 50
+          size: [5, 9]
+          map:
+              -----
+              -+++-
+              -----
+              -----
+              -----
+              -----
+              -----
+              -+++-
+              -----
+    - name: "Room #2"
+      pos: [1050, 0, 0]
+      size: [1000, 1000, 500]
+      height: 500
+      stairtype: none
+      display: []
+      pathfinding:
+          tilesize: 50
+          size: [24, 24]
+          map:
+              -----------------------
+              -+++++++++++++++++++++-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              ---------------------+-
+              ---------------------+-
+              ---------------------+-
+              ---------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+++++++++++++++++++++-
+              -----------------------
+exits:
+    - room1: "Room #1"
+      room2: "Room #2"
+      dir: e
+      pos: [400, 600]
+...

token.h

@@ -5,10 +5,12 @@
 namespace YAML
 {
 	struct Token {
+		Token(): isValid(true), isPossible(true) {}
 		virtual ~Token() {}
 		virtual void Write(std::ostream& out) const {}
 
 		friend std::ostream& operator << (std::ostream& out, const Token& token) { token.Write(out); return out; }
+		bool isValid, isPossible;
 	};
 
 	struct StreamStartToken: public Token {};

yaml-reader.vcproj

@@ -209,6 +209,10 @@
 				RelativePath=".\sequence.cpp"
 				>
 			</File>
+			<File
+				RelativePath=".\simplekey.cpp"
+				>
+			</File>
 		</Filter>
 		<Filter
 			Name="Header Files"