redesigned preprocessor and lexer

README.md

@@ -96,7 +96,7 @@ will be given to issues which include examples or test cases.
 
 ## SystemVerilog Front End
 
-This project contains a preprocessor and lexer, a parser, and an abstract syntax
+This project contains a preprocessor, lexer, and parser, and an abstract syntax
 tree representation for a subset of the SystemVerilog specification. The parser
 is not very strict. The AST allows for the representation of syntactically (and
 semantically) invalid Verilog. The goal is to be more general in the

src/Language/SystemVerilog/Parser.hs

@@ -9,8 +9,9 @@ import Control.Monad.Except
 import Control.Monad.State
 import qualified Data.Map.Strict as Map
 import Language.SystemVerilog.AST (AST)
-import Language.SystemVerilog.Parser.Lex (lexFile, Env)
+import Language.SystemVerilog.Parser.Lex (lexStr)
 import Language.SystemVerilog.Parser.Parse (parse)
+import Language.SystemVerilog.Parser.Preprocess (preprocess, Env)
 import Language.SystemVerilog.Parser.Tokens (Position(..), tokenPosition)
 
 -- parses a compilation unit given include search paths and predefined macros
@@ -32,8 +33,10 @@ parseFiles' includePaths env siloed (path : paths) = do
 -- the file path
 parseFile' :: [String] -> Env -> FilePath -> ExceptT String IO (AST, Env)
 parseFile' includePaths env path = do
-    result <- liftIO $ lexFile includePaths env path
-    (tokens, env') <- liftEither result
+    preResult <- liftIO $ preprocess includePaths env path
+    (contents, env') <- liftEither preResult
+    result <- liftIO $ uncurry lexStr $ unzip contents
+    tokens <- liftEither result
     let position =
             if null tokens
                 then Position path 1 1

src/Language/SystemVerilog/Parser/Parse.y

@@ -297,7 +297,15 @@ systemIdentifier   { Token Id_system       _ _ }
 number             { Token Lit_number      _ _ }
 string             { Token Lit_string      _ _ }
 time               { Token Lit_time        _ _ }
-directive          { Token Spe_Directive   _ _ }
+
+"`celldefine"          { Token Dir_celldefine _ _ }
+"`endcelldefine"       { Token Dir_endcelldefine _ _ }
+"`unconnected_drive"   { Token Dir_unconnected_drive _ _ }
+"`nounconnected_drive" { Token Dir_nounconnected_drive _ _ }
+"`default_nettype"     { Token Dir_default_nettype _ _ }
+"`resetall"            { Token Dir_resetall _ _ }
+"`begin_keywords"      { Token Dir_begin_keywords _ _ }
+"`end_keywords"        { Token Dir_end_keywords _ _ }
 
 "("                { Token Sym_paren_l _ _ }
 ")"                { Token Sym_paren_r _ _ }
@@ -797,7 +805,18 @@ TimeunitsDeclaration :: { [PackageItem] }
   | "timeprecision" Time     ";" { [] }
 
 Directive :: { String }
-  : directive { tokenString $1 }
+  : "`celldefine"          { tokenString $1 }
+  | "`endcelldefine"       { tokenString $1 }
+  | "`unconnected_drive" Drive { tokenString $1 ++ " " ++ $2 }
+  | "`nounconnected_drive" { tokenString $1 }
+  | "`default_nettype" DefaultNetType { tokenString $1 ++ " " ++ $2 }
+  | "`resetall"            { tokenString $1 }
+Drive :: { String }
+  : "pull0" { tokenString $1 }
+  | "pull1" { tokenString $1 }
+DefaultNetType :: { String }
+  : NetType { show $1 }
+  | Identifier { $1 }
 
 PackageImportItems :: { [(Identifier, Maybe Identifier)] }
   : PackageImportItem                        { [$1] }

src/Language/SystemVerilog/Parser/Tokens.hs

@@ -28,7 +28,7 @@ tokenPosition :: Token -> Position
 tokenPosition (Token _ _ pos) = pos
 
 pattern TokenEOF :: Token
-pattern TokenEOF = Token MacroBoundary "" (Position "" 0 0)
+pattern TokenEOF = Token Unknown "" (Position "" 0 0)
 
 data Position
     = Position String Int Int
@@ -391,7 +391,13 @@ data TokenName
     | Sym_amp_amp_amp
     | Sym_lt_lt_lt_eq
     | Sym_gt_gt_gt_eq
-    | Spe_Directive
+    | Dir_celldefine
+    | Dir_endcelldefine
+    | Dir_unconnected_drive
+    | Dir_nounconnected_drive
+    | Dir_default_nettype
+    | Dir_resetall
+    | Dir_begin_keywords
+    | Dir_end_keywords
     | Unknown
-    | MacroBoundary
     deriving (Show, Eq, Ord)

sv2v.cabal

@@ -53,6 +53,7 @@ executable sv2v
     Language.SystemVerilog.Parser.Lex
     Language.SystemVerilog.Parser.Parse
     Language.SystemVerilog.Parser.ParseDecl
+    Language.SystemVerilog.Parser.Preprocess
     Language.SystemVerilog.Parser.Tokens
     -- Conversion modules
     Convert

test/lex/macro_boundary.sv

 `define SIZE 4
 `define NESTED_SIZE `SIZE
 `define NAME op
-module t`NAME;
-    initial $display(`SIZE'ha);
-    initial $display(`NESTED_SIZE'ha);
+`define FOO ha
+`define BAR 'ha
+`define MULTI 1, 2, 5
+`define DULE dule
+mo`DULE t`NAME;
+    initial $display("%b", `SIZE'ha);
+    initial $display("%b", `NESTED_SIZE'ha);
+    initial $display("%b", 10'h`NESTED_SIZE);
+    initial $display("%b", 10`BAR);
+    initial $display("%b", 10`SIZE);
+    initial $display("%b %b %b", `MULTI'ha);
+
+    initial begin : block_name
+        reg [4:0] foo;
+        foo <= #1 `SIZE;
+        $display("%b", foo);
+        #2;
+        $display("%b", foo);
+    end
 endmodule