Merged r295:305 from the tags branch to the trunk

include/emitter.h

@@ -52,6 +52,7 @@ namespace YAML
 		Emitter& Write(double d);
 		Emitter& Write(const _Alias& alias);
 		Emitter& Write(const _Anchor& anchor);
+		Emitter& Write(const _Tag& tag);
 		Emitter& Write(const _Comment& comment);
 		Emitter& Write(const _Null& null);
 		

include/emittermanip.h

@@ -80,6 +80,16 @@ namespace YAML
 	inline _Anchor Anchor(const std::string content) {
 		return _Anchor(content);
 	}
+	
+	struct _Tag {
+		_Tag(const std::string& content_): content(content_), verbatim(true) {}
+		std::string content;
+		bool verbatim;
+	};
+	
+	inline _Tag VerbatimTag(const std::string& content) {
+		return _Tag(content);
+	}
 
 	struct _Comment {
 		_Comment(const std::string& content_): content(content_) {}

include/exceptions.h

@@ -17,7 +17,12 @@ namespace YAML
 		const std::string YAML_DIRECTIVE_ARGS    = "YAML directives must have exactly one argument";
 		const std::string YAML_VERSION           = "bad YAML version: ";
 		const std::string YAML_MAJOR_VERSION     = "YAML major version too large";
+		const std::string REPEATED_YAML_DIRECTIVE= "repeated YAML directive";
 		const std::string TAG_DIRECTIVE_ARGS     = "TAG directives must have exactly two arguments";
+		const std::string REPEATED_TAG_DIRECTIVE = "repeated TAG directive";
+		const std::string CHAR_IN_TAG_HANDLE     = "illegal character found while scanning tag handle";
+		const std::string TAG_WITH_NO_SUFFIX     = "tag handle with no suffix";
+		const std::string END_OF_VERBATIM_TAG    = "end of verbatim tag not found";
 		const std::string END_OF_MAP             = "end of map not found";
 		const std::string END_OF_MAP_FLOW        = "end of map flow not found";
 		const std::string END_OF_SEQ             = "end of sequence not found";
@@ -57,6 +62,7 @@ namespace YAML
 		const std::string SINGLE_QUOTED_CHAR     = "invalid character in single-quoted string";
 		const std::string INVALID_ANCHOR         = "invalid anchor";
 		const std::string INVALID_ALIAS          = "invalid alias";
+		const std::string INVALID_TAG            = "invalid tag";
 		const std::string EXPECTED_KEY_TOKEN     = "expected key token";
 		const std::string EXPECTED_VALUE_TOKEN   = "expected value token";
 		const std::string UNEXPECTED_KEY_TOKEN   = "unexpected key token";

include/node.h

@@ -9,7 +9,6 @@
 #include "iterator.h"
 #include "mark.h"
 #include "noncopyable.h"
-#include "parserstate.h"
 #include <iostream>
 #include <string>
 #include <vector>
@@ -21,6 +20,7 @@ namespace YAML
 	class Content;
 	class Scanner;
 	class Emitter;
+	class ParserState;
 
 	enum CONTENT_TYPE { CT_NONE, CT_SCALAR, CT_SEQUENCE, CT_MAP };
 
@@ -75,6 +75,9 @@ namespace YAML
 		const Node *Identity() const { return m_pIdentity; }
 		bool IsAlias() const { return m_alias; }
 		bool IsReferenced() const { return m_referenced; }
+		
+		// for tags
+		const std::string GetTag() const { return m_tag; } // TODO: should an aliased node return its alias's tag?
 
 		// emitting
 		friend Emitter& operator << (Emitter& out, const Node& node);

include/parser.h

@@ -5,7 +5,6 @@
 
 
 #include "node.h"
-#include "parserstate.h"
 #include "noncopyable.h"
 #include <ios>
 #include <string>
@@ -16,6 +15,7 @@
 namespace YAML
 {
 	class Scanner;
+	class ParserState;
 	struct Token;
 
 	class Parser: private noncopyable
@@ -33,13 +33,13 @@ namespace YAML
 
 	private:
 		void ParseDirectives();
-		void HandleDirective(Token *pToken);
-		void HandleYamlDirective(Token *pToken);
-		void HandleTagDirective(Token *pToken);
+		void HandleDirective(const Token& token);
+		void HandleYamlDirective(const Token& token);
+		void HandleTagDirective(const Token& token);
 
 	private:
 		std::auto_ptr<Scanner> m_pScanner;
-		ParserState m_state;
+		std::auto_ptr<ParserState> m_pState;
 	};
 }
 

src/emitter.cpp

@@ -146,6 +146,8 @@ namespace YAML
 		switch(curState) {
 				// document-level
 			case ES_WAITING_FOR_DOC:
+				m_stream << "---";
+				m_pState->RequireSeparation();
 				m_pState->SwitchState(ES_WRITING_DOC);
 				return true;
 			case ES_WRITING_DOC:
@@ -323,7 +325,10 @@ namespace YAML
 		EMITTER_STATE curState = m_pState->GetCurState();
 		EMITTER_MANIP flowType = m_pState->GetFlowType(GT_SEQ);
 		if(flowType == Block) {
-			if(curState == ES_WRITING_BLOCK_SEQ_ENTRY || curState == ES_WRITING_BLOCK_MAP_KEY || curState == ES_WRITING_BLOCK_MAP_VALUE) {
+			if(curState == ES_WRITING_BLOCK_SEQ_ENTRY ||
+			   curState == ES_WRITING_BLOCK_MAP_KEY || curState == ES_WRITING_BLOCK_MAP_VALUE ||
+			   curState == ES_WRITING_DOC
+			) {
 				m_stream << "\n";
 				m_pState->UnsetSeparation();
 			}
@@ -354,8 +359,12 @@ namespace YAML
 			//       to a flow sequence if it is
 			assert(curState == ES_DONE_WITH_BLOCK_SEQ_ENTRY || curState == ES_WAITING_FOR_BLOCK_SEQ_ENTRY);
 			if(curState == ES_WAITING_FOR_BLOCK_SEQ_ENTRY) {
+				// Note: only one of these will actually output anything for a given situation
+				EmitSeparationIfNecessary();
 				unsigned curIndent = m_pState->GetCurIndent();
-				m_stream << IndentTo(curIndent) << "[]";
+				m_stream << IndentTo(curIndent);
+
+				m_stream << "[]";
 			}
 		} else if(flowType == FT_FLOW) {
 			// Note: flow sequences are allowed to be empty
@@ -384,7 +393,10 @@ namespace YAML
 		EMITTER_STATE curState = m_pState->GetCurState();
 		EMITTER_MANIP flowType = m_pState->GetFlowType(GT_MAP);
 		if(flowType == Block) {
-			if(curState == ES_WRITING_BLOCK_SEQ_ENTRY || curState == ES_WRITING_BLOCK_MAP_KEY || curState == ES_WRITING_BLOCK_MAP_VALUE) {
+			if(curState == ES_WRITING_BLOCK_SEQ_ENTRY ||
+			   curState == ES_WRITING_BLOCK_MAP_KEY || curState == ES_WRITING_BLOCK_MAP_VALUE ||
+			   curState == ES_WRITING_DOC
+			) {
 				m_stream << "\n";
 				m_pState->UnsetSeparation();
 			}
@@ -415,8 +427,11 @@ namespace YAML
 			//       to a flow sequence if it is
 			assert(curState == ES_DONE_WITH_BLOCK_MAP_VALUE || curState == ES_WAITING_FOR_BLOCK_MAP_ENTRY);
 			if(curState == ES_WAITING_FOR_BLOCK_MAP_ENTRY) {
+				// Note: only one of these will actually output anything for a given situation
+				EmitSeparationIfNecessary();
 				unsigned curIndent = m_pState->GetCurIndent();
-				m_stream << IndentTo(curIndent) << "{}";
+				m_stream << IndentTo(curIndent);
+				m_stream << "{}";
 			}
 		} else if(flowType == FT_FLOW) {
 			// Note: flow maps are allowed to be empty
@@ -675,6 +690,22 @@ namespace YAML
 		return *this;
 	}
 	
+	Emitter& Emitter::Write(const _Tag& tag)
+	{
+		if(!good())
+			return *this;
+		
+		PreAtomicWrite();
+		EmitSeparationIfNecessary();
+		if(!Utils::WriteTag(m_stream, tag.content)) {
+			m_pState->SetError(ErrorMsg::INVALID_TAG);
+			return *this;
+		}
+		m_pState->RequireSeparation();
+		// Note: no PostAtomicWrite() because we need another value for this node
+		return *this;
+	}
+
 	Emitter& Emitter::Write(const _Comment& comment)
 	{
 		if(!good())

src/emitterutils.cpp

@@ -293,6 +293,24 @@ namespace YAML
 			out << "&";
 			return WriteAliasName(out, str);
 		}
+
+		bool WriteTag(ostream& out, const std::string& str)
+		{
+			out << "!<";
+			StringCharSource buffer(str.c_str(), str.size());
+			while(buffer) {
+				int n = Exp::URI.Match(buffer);
+				if(n <= 0)
+					return false;
+
+				while(--n >= 0) {
+					out << buffer[0];
+					++buffer;
+				}
+			}
+			out << ">";
+			return true;
+		}
 	}
 }
 

src/emitterutils.h

@@ -18,6 +18,7 @@ namespace YAML
 		bool WriteComment(ostream& out, const std::string& str, int postCommentIndent);
 		bool WriteAlias(ostream& out, const std::string& str);
 		bool WriteAnchor(ostream& out, const std::string& str);
+		bool WriteTag(ostream& out, const std::string& str);
 	}
 }
 

src/exp.h

@@ -25,6 +25,7 @@ namespace YAML
 		const RegEx Digit = RegEx('0', '9');
 		const RegEx Alpha = RegEx('a', 'z') || RegEx('A', 'Z');
 		const RegEx AlphaNumeric = Alpha || Digit;
+		const RegEx Word = AlphaNumeric || RegEx('-');
 		const RegEx Hex = Digit || RegEx('A', 'F') || RegEx('a', 'f');
 		// Valid Unicode code points that are not part of c-printable (YAML 1.2, sec. 5.1)
 		const RegEx NotPrintable = RegEx(0) || 
@@ -45,6 +46,8 @@ namespace YAML
 		            ValueInFlow = RegEx(':') + (BlankOrBreak || RegEx(",}", REGEX_OR));
 		const RegEx Comment = RegEx('#');
 		const RegEx AnchorEnd = RegEx("?:,]}%@`", REGEX_OR) || BlankOrBreak;
+		const RegEx URI = Word || RegEx("#;/?:@&=+$,_.!~*'()[]", REGEX_OR) || (RegEx('%') + Hex + Hex);
+		const RegEx Tag = Word || RegEx("#;/?:@&=+$_.~*'", REGEX_OR) || (RegEx('%') + Hex + Hex);
 
 		// Plain scalar rules:
 		// . Cannot start with a blank.
@@ -79,6 +82,8 @@ namespace YAML
 		const char Tag = '!';
 		const char LiteralScalar = '|';
 		const char FoldedScalar = '>';
+		const char VerbatimTagStart = '<';
+		const char VerbatimTagEnd = '>';
 	}
 }
 

src/node.cpp

@@ -9,6 +9,7 @@
 #include "aliascontent.h"
 #include "iterpriv.h"
 #include "emitter.h"
+#include "tag.h"
 #include <stdexcept>
 
 namespace YAML
@@ -137,10 +138,8 @@ namespace YAML
 		if(m_tag != "")
 			throw ParserException(token.mark, ErrorMsg::MULTIPLE_TAGS);
 
-		m_tag = state.TranslateTag(token.value);
-
-		for(std::size_t i=0;i<token.params.size();i++)
-			m_tag += token.params[i];
+		Tag tag(token);
+		m_tag = tag.Translate(state);
 		pScanner->pop();
 	}
 	
@@ -241,7 +240,10 @@ namespace YAML
 	bool Node::GetScalar(std::string& s) const
 	{
 		if(!m_pContent) {
-			s = "~";
+			if(m_tag.empty())
+				s = "~";
+			else
+				s = "";
 			return true;
 		}
 		
@@ -258,7 +260,8 @@ namespace YAML
 				out << Anchor(node.m_anchor);
 		}
 
-		// TODO: write tag
+		if(node.m_tag != "")
+			out << VerbatimTag(node.m_tag);
 
 		// write content
 		if(node.m_pContent)

src/parser.cpp

@@ -2,6 +2,7 @@
 #include "scanner.h"
 #include "token.h"
 #include "exceptions.h"
+#include "parserstate.h"
 #include <sstream>
 #include <cstdio>
 
@@ -28,7 +29,7 @@ namespace YAML
 	void Parser::Load(std::istream& in)
 	{
 		m_pScanner.reset(new Scanner(in));
-		m_state.Reset();
+		m_pState.reset(new ParserState);
 	}
 
 	// GetNextDocument
@@ -54,7 +55,7 @@ namespace YAML
 			m_pScanner->pop();
 
 		// now parse our root node
-		document.Parse(m_pScanner.get(), m_state);
+		document.Parse(m_pScanner.get(), *m_pState);
 
 		// and finally eat any doc ends we see
 		while(!m_pScanner->empty() && m_pScanner->peek().type == Token::DOC_END)
@@ -83,51 +84,59 @@ namespace YAML
 			// we keep the directives from the last document if none are specified;
 			// but if any directives are specific, then we reset them
 			if(!readDirective)
-				m_state.Reset();
+				m_pState.reset(new ParserState);
 
 			readDirective = true;
-			HandleDirective(&token);
+			HandleDirective(token);
 			m_pScanner->pop();
 		}
 	}
 
-	void Parser::HandleDirective(Token *pToken)
+	void Parser::HandleDirective(const Token& token)
 	{
-		if(pToken->value == "YAML")
-			HandleYamlDirective(pToken);
-		else if(pToken->value == "TAG")
-			HandleTagDirective(pToken);
+		if(token.value == "YAML")
+			HandleYamlDirective(token);
+		else if(token.value == "TAG")
+			HandleTagDirective(token);
 	}
 
 	// HandleYamlDirective
 	// . Should be of the form 'major.minor' (like a version number)
-	void Parser::HandleYamlDirective(Token *pToken)
+	void Parser::HandleYamlDirective(const Token& token)
 	{
-		if(pToken->params.size() != 1)
-			throw ParserException(pToken->mark, ErrorMsg::YAML_DIRECTIVE_ARGS);
+		if(token.params.size() != 1)
+			throw ParserException(token.mark, ErrorMsg::YAML_DIRECTIVE_ARGS);
+		
+		if(!m_pState->version.isDefault)
+			throw ParserException(token.mark, ErrorMsg::REPEATED_YAML_DIRECTIVE);
 
-		std::stringstream str(pToken->params[0]);
-		str >> m_state.version.major;
+		std::stringstream str(token.params[0]);
+		str >> m_pState->version.major;
 		str.get();
-		str >> m_state.version.minor;
+		str >> m_pState->version.minor;
 		if(!str || str.peek() != EOF)
-			throw ParserException(pToken->mark, ErrorMsg::YAML_VERSION + pToken->params[0]);
+			throw ParserException(token.mark, ErrorMsg::YAML_VERSION + token.params[0]);
 
-		if(m_state.version.major > 1)
-			throw ParserException(pToken->mark, ErrorMsg::YAML_MAJOR_VERSION);
+		if(m_pState->version.major > 1)
+			throw ParserException(token.mark, ErrorMsg::YAML_MAJOR_VERSION);
 
+		m_pState->version.isDefault = false;
 		// TODO: warning on major == 1, minor > 2?
 	}
 
 	// HandleTagDirective
 	// . Should be of the form 'handle prefix', where 'handle' is converted to 'prefix' in the file.
-	void Parser::HandleTagDirective(Token *pToken)
+	void Parser::HandleTagDirective(const Token& token)
 	{
-		if(pToken->params.size() != 2)
-			throw ParserException(pToken->mark, ErrorMsg::TAG_DIRECTIVE_ARGS);
+		if(token.params.size() != 2)
+			throw ParserException(token.mark, ErrorMsg::TAG_DIRECTIVE_ARGS);
 
-		std::string handle = pToken->params[0], prefix = pToken->params[1];
-		m_state.tags[handle] = prefix;
+		const std::string& handle = token.params[0];
+		const std::string& prefix = token.params[1];
+		if(m_pState->tags.find(handle) != m_pState->tags.end())
+			throw ParserException(token.mark, ErrorMsg::REPEATED_TAG_DIRECTIVE);
+		
+		m_pState->tags[handle] = prefix;
 	}
 
 	void Parser::PrintTokens(std::ostream& out)

src/parserstate.cpp

@@ -2,23 +2,22 @@
 
 namespace YAML
 {
-	void ParserState::Reset()
+	ParserState::ParserState()
 	{
 		// version
+		version.isDefault = true;
 		version.major = 1;
 		version.minor = 2;
-
-		// and tags
-		tags.clear();
-		tags["!"] = "!";
-		tags["!!"] = "tag:yaml.org,2002:";
 	}
 
-	std::string ParserState::TranslateTag(const std::string& handle) const
+	const std::string ParserState::TranslateTagHandle(const std::string& handle) const
 	{
 		std::map <std::string, std::string>::const_iterator it = tags.find(handle);
-		if(it == tags.end())
+		if(it == tags.end()) {
+			if(handle == "!!")
+				return "tag:yaml.org,2002:";
 			return handle;
+		}
 
 		return it->second;
 	}

include/parserstate.h → src/parserstate.h

@@ -10,16 +10,17 @@
 namespace YAML
 {
 	struct Version {
+		bool isDefault;
 		int major, minor;
 	};
-
+	
 	struct ParserState
 	{
+		ParserState();
+		const std::string TranslateTagHandle(const std::string& handle) const;
+	
 		Version version;
 		std::map <std::string, std::string> tags;
-
-		void Reset();
-		std::string TranslateTag(const std::string& handle) const;
 	};
 }
 

src/regex.h

@@ -37,12 +37,12 @@ namespace YAML
 
 		int Match(const std::string& str) const;
 		int Match(const Stream& in) const;
+		template <typename Source> int Match(const Source& source) const;
 
 	private:
 		RegEx(REGEX_OP op);
 		
 		template <typename Source> bool IsValidSource(const Source& source) const;
-		template <typename Source> int Match(const Source& source) const;
 		template <typename Source> int MatchUnchecked(const Source& source) const;
 
 		template <typename Source> int MatchOpEmpty(const Source& source) const;

src/scanner.cpp

@@ -3,6 +3,7 @@
 #include "exceptions.h"
 #include "exp.h"
 #include <cassert>
+#include <memory>
 
 namespace YAML
 {
@@ -13,6 +14,9 @@ namespace YAML
 
 	Scanner::~Scanner()
 	{
+		for(unsigned i=0;i<m_indentRefs.size();i++)
+			delete m_indentRefs[i];
+		m_indentRefs.clear();
 	}
 
 	// empty
@@ -228,7 +232,9 @@ namespace YAML
 	{
 		m_startedStream = true;
 		m_simpleKeyAllowed = true;
-		m_indents.push(IndentMarker(-1, IndentMarker::NONE));
+		IndentMarker *pIndent = new IndentMarker(-1, IndentMarker::NONE);
+		m_indentRefs.push_back(pIndent);
+		m_indents.push(pIndent);
 		m_anchors.clear();
 	}
 
@@ -257,8 +263,9 @@ namespace YAML
 		if(InFlowContext())
 			return 0;
 		
-		IndentMarker indent(column, type);
-		const IndentMarker& lastIndent = m_indents.top();
+		std::auto_ptr<IndentMarker> pIndent(new IndentMarker(column, type));
+		IndentMarker& indent = *pIndent;
+		const IndentMarker& lastIndent = *m_indents.top();
 
 		// is this actually an indentation?
 		if(indent.column < lastIndent.column)
@@ -276,13 +283,15 @@ namespace YAML
 		indent.pStartToken = &m_tokens.back();
 
 		// and then the indent
-		m_indents.push(indent);
-		return &m_indents.top();
+		m_indents.push(&indent);
+		m_indentRefs.push_back(pIndent.release());
+		return m_indentRefs.back();
 	}
 
 	// PopIndentToHere
 	// . Pops indentations off the stack until we reach the current indentation level,
 	//   and enqueues the proper token each time.
+	// . Then pops all invalid indentations off.
 	void Scanner::PopIndentToHere()
 	{
 		// are we in flow?
@@ -291,7 +300,7 @@ namespace YAML
 
 		// now pop away
 		while(!m_indents.empty()) {
-			const IndentMarker& indent = m_indents.top();
+			const IndentMarker& indent = *m_indents.top();
 			if(indent.column < INPUT.column())
 				break;
 			if(indent.column == INPUT.column() && !(indent.type == IndentMarker::SEQ && !Exp::BlockEntry.Matches(INPUT)))
@@ -299,6 +308,9 @@ namespace YAML
 				
 			PopIndent();
 		}
+		
+		while(!m_indents.empty() && m_indents.top()->status == IndentMarker::INVALID)
+			PopIndent();
 	}
 	
 	// PopAllIndents
@@ -312,7 +324,7 @@ namespace YAML
 
 		// now pop away
 		while(!m_indents.empty()) {
-			const IndentMarker& indent = m_indents.top();
+			const IndentMarker& indent = *m_indents.top();
 			if(indent.type == IndentMarker::NONE)
 				break;
 			
@@ -324,17 +336,17 @@ namespace YAML
 	// . Pops a single indent, pushing the proper token
 	void Scanner::PopIndent()
 	{
-		IndentMarker indent = m_indents.top();
-		IndentMarker::INDENT_TYPE type = indent.type;
+		const IndentMarker& indent = *m_indents.top();
 		m_indents.pop();
-		if(!indent.isValid) {
+
+		if(indent.status != IndentMarker::VALID) {
 			InvalidateSimpleKey();
 			return;
 		}
 		
-		if(type == IndentMarker::SEQ)
+		if(indent.type == IndentMarker::SEQ)
 			m_tokens.push(Token(Token::BLOCK_SEQ_END, INPUT.mark()));
-		else if(type == IndentMarker::MAP)
+		else if(indent.type == IndentMarker::MAP)
 			m_tokens.push(Token(Token::BLOCK_MAP_END, INPUT.mark()));
 	}
 
@@ -343,7 +355,7 @@ namespace YAML
 	{
 		if(m_indents.empty())
 			return 0;
-		return m_indents.top().column;
+		return m_indents.top()->column;
 	}
 
 	// Save

src/scanner.h

@@ -36,11 +36,12 @@ namespace YAML
 	private:
 		struct IndentMarker {
 			enum INDENT_TYPE { MAP, SEQ, NONE };
-			IndentMarker(int column_, INDENT_TYPE type_): column(column_), type(type_), isValid(true), pStartToken(0) {}
+			enum STATUS { VALID, INVALID, UNKNOWN };
+			IndentMarker(int column_, INDENT_TYPE type_): column(column_), type(type_), status(VALID), pStartToken(0) {}
 		
 			int column;
 			INDENT_TYPE type;
-			bool isValid;
+			STATUS status;
 			Token *pStartToken;
 		};
 		
@@ -118,10 +119,12 @@ namespace YAML
 		bool m_startedStream, m_endedStream;
 		bool m_simpleKeyAllowed;
 		std::stack <SimpleKey> m_simpleKeys;
-		std::stack <IndentMarker> m_indents;
+		std::stack <IndentMarker *> m_indents;
+		std::vector <IndentMarker *> m_indentRefs; // for "garbage collection"
 		std::stack <FLOW_MARKER> m_flows;
 		std::map <std::string, const Node *> m_anchors;
 	};
 }
 
 #endif // SCANNER_H_62B23520_7C8E_11DE_8A39_0800200C9A66
+

src/scanscalar.h

@@ -40,3 +40,4 @@ namespace YAML
 }
 
 #endif // SCANSCALAR_H_62B23520_7C8E_11DE_8A39_0800200C9A66
+

src/scantag.cpp

+#include "scanner.h"
+#include "regex.h"
+#include "exp.h"
+#include "exceptions.h"
+
+namespace YAML
+{
+	const std::string ScanVerbatimTag(Stream& INPUT)
+	{
+		std::string tag;
+		
+		// eat the start character
+		INPUT.get();
+		
+		while(INPUT) {
+			if(INPUT.peek() == Keys::VerbatimTagEnd) {
+				// eat the end character
+				INPUT.get();
+				return tag;
+			}
+		
+			int n = Exp::URI.Match(INPUT);
+			if(n <= 0)
+				break;
+			
+			tag += INPUT.get(n);
+		}
+
+		throw ParserException(INPUT.mark(), ErrorMsg::END_OF_VERBATIM_TAG);
+	}
+	
+	const std::string ScanTagHandle(Stream& INPUT, bool& canBeHandle)
+	{
+		std::string tag;
+		canBeHandle = true;
+		Mark firstNonWordChar;
+		
+		while(INPUT) {
+			if(INPUT.peek() == Keys::Tag) {
+				if(!canBeHandle)
+					throw ParserException(firstNonWordChar, ErrorMsg::CHAR_IN_TAG_HANDLE);
+				break;
+			}
+
+			int n = 0;
+			if(canBeHandle) {
+				n = Exp::Word.Match(INPUT);
+				if(n <= 0) {
+					canBeHandle = false;
+					firstNonWordChar = INPUT.mark();
+				}
+			}
+			
+			if(!canBeHandle)
+				n = Exp::Tag.Match(INPUT);
+
+			if(n <= 0)
+				break;
+			
+			tag += INPUT.get(n);
+		}
+
+		return tag;
+	}
+	
+	const std::string ScanTagSuffix(Stream& INPUT)
+	{
+		std::string tag;
+		
+		while(INPUT) {
+			int n = Exp::Tag.Match(INPUT);
+			if(n <= 0)
+				break;
+			
+			tag += INPUT.get(n);
+		}
+		
+		if(tag.empty())
+			throw ParserException(INPUT.mark(), ErrorMsg::TAG_WITH_NO_SUFFIX);
+		
+		return tag;
+	}
+}
+

src/scantag.h

+#pragma once
+
+#ifndef SCANTAG_H_62B23520_7C8E_11DE_8A39_0800200C9A66
+#define SCANTAG_H_62B23520_7C8E_11DE_8A39_0800200C9A66
+
+
+#include <string>
+#include "stream.h"
+
+namespace YAML
+{
+	const std::string ScanVerbatimTag(Stream& INPUT);
+	const std::string ScanTagHandle(Stream& INPUT, bool& canBeHandle);
+	const std::string ScanTagSuffix(Stream& INPUT);
+}
+
+#endif // SCANTAG_H_62B23520_7C8E_11DE_8A39_0800200C9A66
+

src/scantoken.cpp

@@ -3,6 +3,8 @@
 #include "exceptions.h"
 #include "exp.h"
 #include "scanscalar.h"
+#include "scantag.h"
+#include "tag.h"
 #include <sstream>
 
 namespace YAML
@@ -24,12 +26,12 @@ namespace YAML
 		m_simpleKeyAllowed = false;
 
 		// store pos and eat indicator
-		Mark mark = INPUT.mark();
+		Token token(Token::DIRECTIVE, INPUT.mark());
 		INPUT.eat(1);
 
 		// read name
 		while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
-			name += INPUT.get();
+			token.value += INPUT.get();
 
 		// read parameters
 		while(1) {
@@ -46,12 +48,9 @@ namespace YAML
 			while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
 				param += INPUT.get();
 
-			params.push_back(param);
+			token.params.push_back(param);
 		}
 		
-		Token token(Token::DIRECTIVE, mark);
-		token.value = name;
-		token.params = params;
 		m_tokens.push(token);
 	}
 
@@ -242,37 +241,34 @@ namespace YAML
 	// Tag
 	void Scanner::ScanTag()
 	{
-		std::string handle, suffix;
-
 		// insert a potential simple key
 		InsertPotentialSimpleKey();
 		m_simpleKeyAllowed = false;
 
-		// eat the indicator
-		Mark mark = INPUT.mark();
-		handle += INPUT.get();
-
-		// read the handle
-		while(INPUT && INPUT.peek() != Keys::Tag && !Exp::BlankOrBreak.Matches(INPUT))
-			handle += INPUT.get();
+		Token token(Token::TAG, INPUT.mark());
 
-		// is there a suffix?
-		if(INPUT.peek() == Keys::Tag) {
-			// eat the indicator
-			handle += INPUT.get();
+		// eat the indicator
+		INPUT.get();
+		
+		if(INPUT && INPUT.peek() == Keys::VerbatimTagStart){
+			std::string tag = ScanVerbatimTag(INPUT);
 
-			// then read it
-			while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
-				suffix += INPUT.get();
+			token.value = tag;
+			token.data = Tag::VERBATIM;
 		} else {
-			// this is a bit weird: we keep just the '!' as the handle and move the rest to the suffix
-			suffix = handle.substr(1);
-			handle = "!";
+			bool canBeHandle;
+			token.value = ScanTagHandle(INPUT, canBeHandle);
+			token.data = (token.value.empty() ? Tag::SECONDARY_HANDLE : Tag::PRIMARY_HANDLE);
+			
+			// is there a suffix?
+			if(canBeHandle && INPUT.peek() == Keys::Tag) {
+				// eat the indicator
+				INPUT.get();
+				token.params.push_back(ScanTagSuffix(INPUT));
+				token.data = Tag::NAMED_HANDLE;
+			}
 		}
 
-		Token token(Token::TAG, mark);
-		token.value = handle;
-		token.params.push_back(suffix);
 		m_tokens.push(token);
 	}
 

src/simplekey.cpp

@@ -12,9 +12,11 @@ namespace YAML
 
 	void Scanner::SimpleKey::Validate()
 	{
-		// Note: pIndent will *not* be garbage here; see below
+		// Note: pIndent will *not* be garbage here;
+		//       we "garbage collect" them so we can
+		//       always refer to them
 		if(pIndent)
-			pIndent->isValid = true;
+			pIndent->status = IndentMarker::VALID;
 		if(pMapStart)
 			pMapStart->status = Token::VALID;
 		if(pKey)
@@ -23,8 +25,8 @@ namespace YAML
 
 	void Scanner::SimpleKey::Invalidate()
 	{
-		// Note: pIndent might be a garbage pointer here, but that's ok
-		//       An indent will only be popped if the simple key is invalid
+		if(pIndent)
+			pIndent->status = IndentMarker::INVALID;
 		if(pMapStart)
 			pMapStart->status = Token::INVALID;
 		if(pKey)
@@ -68,7 +70,7 @@ namespace YAML
 		// first add a map start, if necessary
 		key.pIndent = PushIndentTo(INPUT.column(), IndentMarker::MAP);
 		if(key.pIndent) {
-			key.pIndent->isValid = false;
+			key.pIndent->status = IndentMarker::UNKNOWN;
 			key.pMapStart = key.pIndent->pStartToken;
 			key.pMapStart->status = Token::UNVERIFIED;
 		}
@@ -135,3 +137,4 @@ namespace YAML
 			m_simpleKeys.pop();
 	}
 }
+

src/stringsource.h

@@ -30,6 +30,11 @@ namespace YAML
 			++m_offset;
 			return *this;
 		}
+		
+		StringCharSource& operator += (std::size_t offset) {
+			m_offset += offset;
+			return *this;
+		}
 	private:
 		const char *m_str;
 		std::size_t m_size;

src/tag.cpp

+#include "tag.h"
+#include "token.h"
+#include "parserstate.h"
+#include <cassert>
+
+namespace YAML
+{
+	Tag::Tag(const Token& token): type(static_cast<TYPE>(token.data))
+	{
+		switch(type) {
+			case VERBATIM:
+				value = token.value;
+				break;
+			case PRIMARY_HANDLE:
+				value = token.value;
+				break;
+			case SECONDARY_HANDLE:
+				value = token.value;
+				break;
+			case NAMED_HANDLE:
+				handle = token.value;
+				value = token.params[0];
+				break;
+			case NON_SPECIFIC:
+				break;
+			default:
+				assert(false);
+		}
+	}
+
+	const std::string Tag::Translate(const ParserState& state)
+	{
+		switch(type) {
+			case VERBATIM:
+				return value;
+			case PRIMARY_HANDLE:
+				return state.TranslateTagHandle("!") + value;
+			case SECONDARY_HANDLE:
+				return state.TranslateTagHandle("!!") + value;
+			case NAMED_HANDLE:
+				return state.TranslateTagHandle("!" + handle + "!") + value;
+			case NON_SPECIFIC:
+				// TODO:
+				return "!";
+			default:
+				assert(false);
+		}
+	}
+}
+

src/tag.h

+#pragma once
+
+#ifndef TAG_H_62B23520_7C8E_11DE_8A39_0800200C9A66
+#define TAG_H_62B23520_7C8E_11DE_8A39_0800200C9A66
+
+#include <string>
+
+namespace YAML
+{
+	struct Token;
+	struct ParserState;
+
+	struct Tag {
+		enum TYPE {
+			VERBATIM, PRIMARY_HANDLE, SECONDARY_HANDLE, NAMED_HANDLE, NON_SPECIFIC
+		};
+		
+		Tag(const Token& token);
+		const std::string Translate(const ParserState& state);
+		
+		TYPE type;
+		std::string handle, value;
+	};
+}
+
+#endif // TAG_H_62B23520_7C8E_11DE_8A39_0800200C9A66

src/token.h

@@ -59,7 +59,7 @@ namespace YAML
 		};
 				
 		// data
-		Token(TYPE type_, const Mark& mark_): status(VALID), type(type_), mark(mark_) {}
+		Token(TYPE type_, const Mark& mark_): status(VALID), type(type_), mark(mark_), data(0) {}
 
 		friend std::ostream& operator << (std::ostream& out, const Token& token) {
 			out << TokenNames[token.type] << std::string(": ") << token.value;
@@ -73,6 +73,7 @@ namespace YAML
 		Mark mark;
 		std::string value;
 		std::vector <std::string> params;
+		int data;
 	};
 }