support for loading from library directories

CHANGELOG.md

 ## Unreleased
 
+### New Features
+
+* Added `-y`/`--libdir` for specifying library directories from which to
+  automatically load modules and interfaces used in the design that are not
+  found in the provided input files
+* The `string` data type is now dropped from parameters and localparams
+* Added support for passing through `sequence` and `property` declarations
+
 ### Bug Fixes
 
 * Fixed crash when converting multi-dimensional arrays or arrays of structs or
@@ -18,11 +26,6 @@
 * Fixed keywords included in the "1364-2001" and "1364-2001-noconfig"
   `begin_keywords` version specifiers
 
-### New Features
-
-* `string` data type is now dropped from parameters and localparams
-* Added support for passing through `sequence` and `property` declarations
-
 ### Other Enhancements
 
 * Added elaboration for accesses to fields of struct constants, which can

README.md

@@ -78,7 +78,9 @@ default. Users should typically pass all of their SystemVerilog source files to
 sv2v at once so it can properly resolve packages, interfaces, type parameters,
 etc., across files. Using `--write=adjacent` will create a converted `.v` for
 every `.sv` input file rather than printing to `stdout`. `--write`/`-w` can also
-be used to specify a path to a `.v` output file.
+be used to specify a path to a `.v` output file. Undefined modules and
+interfaces can be automatically loaded from library directories using
+`--libdir`/`-y`.
 
 Users may specify `include` search paths, define macros during preprocessing,
 and exclude some of the conversions. Specifying `-` as an input file will read
@@ -91,6 +93,8 @@ sv2v [OPTIONS] [FILES]
 
 Preprocessing:
   -I --incdir=DIR           Add directory to include search path
+  -y --libdir=DIR           Add a directory to the library search path used
+                            when looking for undefined modules and interfaces
   -D --define=NAME[=VALUE]  Define a macro for preprocessing
      --siloed               Lex input files separately, so macros from
                             earlier files are not defined in later files

src/Job.hs

@@ -36,6 +36,7 @@ data Write
 data Job = Job
     { files :: [FilePath]
     , incdir :: [FilePath]
+    , libdir :: [FilePath]
     , define :: [String]
     , siloed :: Bool
     , skipPreprocessor :: Bool
@@ -58,6 +59,9 @@ defaultJob = Job
     , incdir = nam_ "I" &= name "incdir" &= typDir
         &= help "Add directory to include search path"
         &= groupname "Preprocessing"
+    , libdir = nam_ "y" &= name "libdir" &= typDir
+        &= help ("Add a directory to the library search path used when looking"
+            ++ " for undefined modules and interfaces")
     , define = nam_ "D" &= name "define" &= typ "NAME[=VALUE]"
         &= help "Define a macro for preprocessing"
     , siloed = nam_ "siloed" &= help ("Lex input files separately, so"

src/Language/SystemVerilog/Parser.hs

@@ -3,28 +3,41 @@
  - Author: Zachary Snow <zach@zachjs.com>
  -}
 module Language.SystemVerilog.Parser
-    ( initialEnv
-    , parseFiles
+    ( parseFiles
     , Config(..)
     ) where
 
 import Control.Monad.Except
 import Data.List (elemIndex)
+import Data.Maybe (catMaybes)
+import System.Directory (findFile)
 import qualified Data.Map.Strict as Map
+import qualified Data.Set as Set
 
 import Language.SystemVerilog.AST (AST)
 import Language.SystemVerilog.Parser.Lex (lexStr)
 import Language.SystemVerilog.Parser.Parse (parse)
 import Language.SystemVerilog.Parser.Preprocess (preprocess, annotate, Env, Contents)
 
+type Output = (FilePath, AST)
+type Strings = Set.Set String
+
 data Config = Config
-    { cfEnv :: Env
+    { cfDefines :: [String]
     , cfIncludePaths :: [FilePath]
+    , cfLibraryPaths :: [FilePath]
     , cfSiloed :: Bool
     , cfSkipPreprocessor :: Bool
     , cfOversizedNumbers :: Bool
     }
 
+data Context = Context
+    { ctConfig :: Config
+    , ctEnv :: Env
+    , ctUsed :: Strings
+    , ctHave :: Strings
+    }
+
 -- parse CLI macro definitions into the internal macro environment format
 initialEnv :: [String] -> Env
 initialEnv = Map.map (, []) . Map.fromList . map splitDefine
@@ -38,26 +51,58 @@ splitDefine str =
             where (name, rest) = splitAt idx str
 
 -- parse a list of files according to the given configuration
-parseFiles :: Config -> [FilePath] -> ExceptT String IO [AST]
-parseFiles _ [] = return []
-parseFiles config (path : paths) = do
-    (config', ast) <- parseFile config path
-    fmap (ast :) $ parseFiles config' paths
-
--- parse an individual file, potentially updating the configuration
-parseFile :: Config -> FilePath -> ExceptT String IO (Config, AST)
-parseFile config path = do
-    (config', contents) <- preprocessFile config path
+parseFiles :: Config -> [FilePath] -> ExceptT String IO [Output]
+parseFiles config = parseFiles' context . zip (repeat "")
+    where
+        context = Context config env mempty mempty
+        env = initialEnv $ cfDefines config
+
+-- parse files, keeping track of which parts are defined and used
+parseFiles' :: Context -> [(String, FilePath)] -> ExceptT String IO [Output]
+
+-- look for missing parts in libraries if any library paths were provided
+parseFiles' context []
+    | null libdirs = return []
+    | otherwise = do
+        possibleFiles <- catMaybes <$> mapM lookupLibrary missingParts
+        if null possibleFiles
+            then return []
+            else parseFiles' context possibleFiles
+    where
+        missingParts = Set.toList $ ctUsed context Set.\\ ctHave context
+        libdirs = cfLibraryPaths $ ctConfig context
+        lookupLibrary partName = ((partName, ) <$>) <$> lookupLibFile partName
+        lookupLibFile = liftIO . findFile libdirs . (++ ".sv")
+
+-- load the files, but complain if an expected part is missing
+parseFiles' context ((part, path) : files) = do
+    (context', ast) <- parseFile context path
+    let misdirected = not $ null part || Set.member part (ctHave context')
+    when misdirected $ throwError $
+        "Expected to find module or interface " ++ show part ++ " in file "
+        ++ show path ++ " selected from the library path."
+    ((path, ast) :) <$> parseFiles' context' files
+
+-- parse an individual file, updating the context
+parseFile :: Context -> FilePath -> ExceptT String IO (Context, AST)
+parseFile context path = do
+    (context', contents) <- preprocessFile context path
     tokens <- liftEither $ runExcept $ lexStr contents
-    ast <- parse (cfOversizedNumbers config) tokens
-    return (config', ast)
-
--- preprocess an individual file, potentially updating the configuration
-preprocessFile :: Config -> FilePath -> ExceptT String IO (Config, Contents)
-preprocessFile config path | cfSkipPreprocessor config =
-    fmap (config, ) $ annotate path
-preprocessFile config path = do
-    (env', contents) <- preprocess (cfIncludePaths config) env path
-    let config' = config { cfEnv = if cfSiloed config then env else env' }
-    return (config', contents)
-    where env = cfEnv config
+    (ast, used, have) <- parse (cfOversizedNumbers config) tokens
+    let context'' = context' { ctUsed = used <> ctUsed context
+                             , ctHave = have <> ctHave context }
+    return (context'', ast)
+    where config = ctConfig context
+
+-- preprocess an individual file, potentially updating the environment
+preprocessFile :: Context -> FilePath -> ExceptT String IO (Context, Contents)
+preprocessFile context path
+    | cfSkipPreprocessor config =
+        (context, ) <$> annotate path
+    | otherwise = do
+        (env', contents) <- preprocess (cfIncludePaths config) env path
+        let context' = context { ctEnv = if cfSiloed config then env else env' }
+        return (context', contents)
+    where
+        config = ctConfig context
+        env = ctEnv context

src/Language/SystemVerilog/Parser/Parse.y

@@ -20,6 +20,7 @@ import Control.Monad.Except
 import Control.Monad.State.Strict
 import Data.Maybe (catMaybes, fromMaybe)
 import System.IO (hPutStrLn, stderr)
+import qualified Data.Set as Set
 import Language.SystemVerilog.AST
 import Language.SystemVerilog.Parser.ParseDecl
 import Language.SystemVerilog.Parser.Tokens
@@ -554,7 +555,7 @@ Part(begin, end) :: { Description }
 
 PartHeader :: { [Attr] -> Bool -> PartKW -> [ModuleItem] -> Identifier -> ParseState Description }
   : Lifetime Identifier PackageImportDeclarations Params PortDecls ";"
-    { \attrs extern kw items label -> checkTag $2 label $ Part attrs extern kw $1 $2 (fst $5) ($3 ++ $4 ++ (snd $5) ++ items) }
+    {% recordPartHave $2 >> return \attrs extern kw items label -> checkTag $2 label $ Part attrs extern kw $1 $2 (fst $5) ($3 ++ $4 ++ (snd $5) ++ items) }
 
 ModuleKW :: { PartKW }
   : "module" { Module }
@@ -701,7 +702,7 @@ ModuleItem :: { [ModuleItem] }
   | "generate" GenItems endgenerate { [Generate $2] }
 NonGenerateModuleItem :: { [ModuleItem] }
   -- This item covers module instantiations and all declarations
-  : ModuleDeclTokens(";")                { parseDTsAsModuleItems $1 }
+  : ModuleDeclTokens(";")                {% mapM recordPartUsed $ parseDTsAsModuleItems $1 }
   | ParameterDecl(";")                   { map (MIPackageItem . Decl) $1 }
   | "defparam" LHSAsgns ";"              { map (uncurry Defparam) $2 }
   | "assign" AssignOption LHSAsgns ";"   { map (uncurry $ Assign $2) $3 }
@@ -1549,25 +1550,29 @@ data ParseData = ParseData
   { pPosition :: Position
   , pTokens :: [Token]
   , pOversizedNumbers :: Bool
+  , pPartsUsed :: Strings
+  , pPartsHave :: Strings
   }
 
 type ParseState = StateT ParseData (ExceptT String IO)
+type Strings = Set.Set String
 
-parse :: Bool -> [Token] -> ExceptT String IO AST
-parse _ [] = return []
-parse oversizedNumbers tokens =
-  evalStateT parseMain initialState
+parse :: Bool -> [Token] -> ExceptT String IO (AST, Strings, Strings)
+parse _ [] = return mempty
+parse oversizedNumbers tokens = do
+  (ast, finalState) <- runStateT parseMain initialState
+  return (ast, pPartsUsed finalState, pPartsHave finalState)
   where
     position = tokenPosition $ head tokens
-    initialState = ParseData position tokens oversizedNumbers
+    initialState = ParseData position tokens oversizedNumbers mempty mempty
 
 positionKeep :: (Token -> ParseState a) -> ParseState a
 positionKeep cont = do
-  ParseData _ tokens oversizedNumbers <- get
+  tokens <- gets pTokens
   case tokens of
     [] -> cont TokenEOF
     tok : toks -> do
-      put $ ParseData (tokenPosition tok) toks oversizedNumbers
+      modify' $ \s -> s { pPosition = tokenPosition tok, pTokens = toks }
       cont tok
 
 parseErrorTok :: Token -> ParseState a
@@ -1842,4 +1847,19 @@ portBindingAttrs (pos, attrs) = parseWarning pos msg
   where msg = "Ignored port connection attributes "
           ++ concatMap show attrs ++ "."
 
+recordPartUsed :: ModuleItem -> ParseState ModuleItem
+recordPartUsed item@(Instance partName _ _ _ _) = do
+  partsUsed <- gets pPartsUsed
+  when (Set.notMember partName partsUsed) $ do
+    let partsUsed' = Set.insert partName partsUsed
+    modify' $ \s -> s { pPartsUsed = partsUsed' }
+  return item
+recordPartUsed item = return item
+
+recordPartHave :: Identifier -> ParseState ()
+recordPartHave partName = do
+  partsHave <- gets pPartsHave
+  let partsHave' = Set.insert partName partsHave
+  modify' $ \s -> s { pPartsHave = partsHave' }
+
 }

src/sv2v.hs

@@ -13,7 +13,7 @@ import Control.Monad.Except (runExceptT)
 import Convert (convert)
 import Job (readJob, Job(..), Write(..))
 import Language.SystemVerilog.AST
-import Language.SystemVerilog.Parser (initialEnv, parseFiles, Config(..))
+import Language.SystemVerilog.Parser (parseFiles, Config(..))
 
 isInterface :: Description -> Bool
 isInterface (Part _ _ Interface _ _ _ _ ) = True
@@ -69,8 +69,9 @@ main = do
     job <- readJob
     -- parse the input files
     let config = Config
-            { cfEnv              = initialEnv (define job)
+            { cfDefines          = define job
             , cfIncludePaths     = incdir job
+            , cfLibraryPaths     = libdir job
             , cfSiloed           = siloed job
             , cfSkipPreprocessor = skipPreprocessor job
             , cfOversizedNumbers = oversizedNumbers job
@@ -80,12 +81,13 @@ main = do
         Left msg -> do
             hPutStrLn stderr msg
             exitFailure
-        Right asts -> do
+        Right inputs -> do
+            let (inPaths, asts) = unzip inputs
             -- convert the files if requested
             asts' <- if passThrough job
                         then return asts
                         else convert (dumpPrefix job) (exclude job) asts
             emptyWarnings (concat asts) (concat asts')
             -- write the converted files out
-            writeOutput (write job) (files job) asts'
+            writeOutput (write job) inPaths asts'
             exitSuccess

test/README.md

@@ -73,6 +73,7 @@ Many of these suites test a particular feature of the sv2v CLI.
 * `help` ensures the `--help` output in the README is up to date
 * `keyword` tests `begin_keywords` version specifiers
 * `number` generates and tests short number literals
+* `search` tests `-y`/`--libdir`
 * `siloed` tests `--siloed` and default compilation unit behavior
 * `truncate` tests number literal truncation and `--oversized-numbers`
 * `warning` tests conversion warnings

test/search/.gitignore

+*.v

test/search/apple.sv

+module apple;
+    initial $display("apple");
+endmodule

test/search/misdirect.sv

+module surprise;
+    initial $display("This isn't what you're looking for!");
+endmodule

test/search/orange.sv

+interface orange;
+    initial $display("orange");
+endinterface

test/search/run.sh

+#!/bin/bash
+
+evaluate() {
+    design_v=$SHUNIT_TMPDIR/search_design.v
+    output_log=$SHUNIT_TMPDIR/search.log
+    touch $output_log
+    simulate /dev/null $output_log top <(echo "$1") /dev/null
+    tail -n1 $output_log
+}
+
+search() {
+    top_sv=$SHUNIT_TMPDIR/search_top.sv
+    echo "module top; $mod m(); endmodule" > $top_sv
+    runAndCapture "$@" $top_sv
+}
+
+searchAndEvaluate() {
+    search "$@"
+    assertTrue "$mod conversion should succeed" $result
+    assertNotNull "$mod stdout should not be empty" "$stdout"
+    assertNull "$mod stderr should be empty" "$stderr"
+    output=`evaluate "$stdout"`
+}
+
+checkFound() {
+    searchAndEvaluate "$@"
+    assertEquals "simulation output should match" $mod "$output"
+}
+
+checkNotFound() {
+    searchAndEvaluate "$@"
+    assertContains "iverilog should fail" "$output" "$mod referenced 1 times"
+}
+
+test_found() {
+    for mod in apple orange; do
+        checkFound -y.
+        checkFound -y../base -y.
+        checkFound -y. -y../base
+    done
+}
+
+test_not_found_default() {
+    for mod in apple orange; do
+        checkNotFound
+    done
+}
+
+test_not_found_missing() {
+    for mod in apple orange doesnt_exist; do
+        checkNotFound -y../base
+    done
+}
+
+test_misdirect() {
+    mod=misdirect
+    runAndCapture -y. <(echo "module top; $mod m(); endmodule")
+    assertFalse "conversion should not succeed" $result
+    assertNull "stdout should be empty" "$stdout"
+    assertContains "stderr should match expected error" "$stderr" \
+        'Expected to find module or interface "misdirect" in file "./misdirect.sv" selected from the library path.'
+}
+
+test_found_write_adjacent() {
+    files=(apple.v orange.v top.v)
+    for file in "${files[@]}"; do
+        assertTrue "$file should not exist" "[ ! -f $file ]"
+    done
+
+    runAndCapture -y. -wadj top.sv
+    assertTrue "conversion should succeed" $result
+    assertNull "stdout should be empty" "$stdout"
+    assertNull "stderr should be empty" "$stderr"
+
+    for file in "${files[@]}"; do
+        assertTrue "$file should exist" "[ -f $file ]"
+        rm -f $file
+    done
+}
+
+source ../lib/functions.sh
+
+. shunit2

test/search/top.sv

+module top;
+    apple a();
+    orange o();
+endmodule