Experimental support for git dependencies

README.md

@@ -209,7 +209,7 @@ Available options:
 Available commands:
   init                     Initialize a Nix project. Existing files won't be
                            modified.
-  add                      Add dependency
+  add                      Add a GitHub dependency
   show                     
   update                   Update dependencies
   modify                   Modify dependency
@@ -226,11 +226,11 @@ Examples:
   niv add NixOS/nixpkgs-channels -n nixpkgs -b nixos-19.03
   niv add my-package -v alpha-0.1 -t http://example.com/archive/<version>.zip
 
-Usage: niv add [-n|--name NAME] PACKAGE ([-a|--attribute KEY=VAL] |
+Usage: niv add PACKAGE [-n|--name NAME] ([-a|--attribute KEY=VAL] |
                [-s|--string-attribute KEY=VAL] | [-b|--branch BRANCH] |
                [-o|--owner OWNER] | [-r|--repo REPO] | [-v|--version VERSION] |
                [-t|--template URL] | [-T|--type TYPE])
-  Add dependency
+  Add a GitHub dependency
 
 Available options:
   -n,--name NAME           Set the package name to <NAME>
@@ -249,6 +249,10 @@ Available options:
                            inferred from the suffix of the URL.
   -h,--help                Show this help text
 
+Experimental commands:
+  git                      Add a git dependency. Experimental.
+  github                   Add a GitHub dependency
+
 ```
 
 #### Update

default.nix

@@ -31,7 +31,9 @@ with rec
     "^src/Data/Aeson$"
     "^src/Data/HashMap$"
     "^src/Data/HashMap/Strict$"
+    "^src/Data/Text$"
     "^src/Niv$"
+    "^src/Niv/Git$"
     "^src/Niv/GitHub$"
     "^src/Niv/Sources$"
     "^src/Niv/Update$"
@@ -188,7 +190,8 @@ rec
 {
   inherit niv niv-sdist niv-source niv-devshell niv-cabal-upload;
 
-  tests = pkgs.callPackage ./tests { inherit niv; };
+  tests-github = pkgs.callPackage ./tests/github { inherit niv; };
+  tests-git = pkgs.callPackage ./tests/git { inherit niv; };
 
   niv-test = pkgs.runCommand "niv-test" { buildInputs = [ niv ]; }
     "niv-test && touch $out";

nix/sources.nix

@@ -18,6 +18,9 @@ let
     else
       pkgs.fetchzip { inherit (spec) url sha256; };
 
+  fetch_git = spec:
+      builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; };
+
   fetch_builtin-tarball = spec:
     builtins.trace
       ''
@@ -80,10 +83,11 @@ let
       abort "ERROR: niv spec ${name} does not have a 'type' attribute"
     else if spec.type == "file" then fetch_file spec
     else if spec.type == "tarball" then fetch_tarball spec
+    else if spec.type == "git" then fetch_git spec
     else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec
     else if spec.type == "builtin-url" then fetch_builtin-url spec
     else
-      abort "ERROR: niv spec ${name} has unknown type ${builtins.fromJSON spec.type}";
+      abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
 
   # Ports of functions for older nix versions
 

src/Data/Text/Extended.hs

@@ -0,0 +1,16 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+module Data.Text.Extended where
+
+import Niv.Logger
+import System.Exit (exitFailure)
+import qualified Data.Text as T
+
+tshow :: Show a => a -> T.Text
+tshow = T.pack . show
+
+-- not quite the perfect place for this
+abort :: T.Text -> IO a
+abort msg = do
+    tsay $ T.unwords [ tbold $ tred "FATAL:", msg ]
+    exitFailure

src/Niv/Cli.hs

@@ -11,22 +11,20 @@ module Niv.Cli where
 import Control.Applicative
 import Control.Monad
 import Data.Aeson ((.=))
-import Data.Bifunctor
-import Data.Maybe
 import Data.Char (isSpace)
-import Data.Functor
 import Data.HashMap.Strict.Extended
 import Data.Hashable (Hashable)
 import Data.String.QQ (s)
+import Data.Text.Extended
 import Data.Version (showVersion)
-import Niv.GitHub
+import Niv.Cmd
+import Niv.Git.Cmd
+import Niv.GitHub.Cmd
 import Niv.Logger
 import Niv.Sources
 import Niv.Update
 import System.Environment (getArgs)
-import System.Exit (ExitCode(ExitSuccess))
 import System.FilePath (takeDirectory)
-import System.Process (readProcessWithExitCode)
 import UnliftIO
 import qualified Data.Aeson as Aeson
 import qualified Data.ByteString as B
@@ -70,72 +68,8 @@ parsePackageName :: Opts.Parser PackageName
 parsePackageName = PackageName <$>
     Opts.argument Opts.str (Opts.metavar "PACKAGE")
 
-parsePackageSpec :: Opts.Parser PackageSpec
-parsePackageSpec =
-    (PackageSpec . HMS.fromList) <$>
-      many parseAttribute
-  where
-    parseAttribute :: Opts.Parser (T.Text, Aeson.Value)
-    parseAttribute =
-      Opts.option (Opts.maybeReader parseKeyValJSON)
-        ( Opts.long "attribute" <>
-          Opts.short 'a' <>
-          Opts.metavar "KEY=VAL" <>
-          Opts.help "Set the package spec attribute <KEY> to <VAL>, where <VAL> may be JSON."
-        ) <|>
-      Opts.option (Opts.maybeReader (parseKeyVal Aeson.toJSON))
-        ( Opts.long "string-attribute" <>
-          Opts.short 's' <>
-          Opts.metavar "KEY=VAL" <>
-          Opts.help "Set the package spec attribute <KEY> to <VAL>."
-        ) <|>
-      shortcutAttributes <|>
-      ((("url_template",) . Aeson.String) <$> Opts.strOption
-        ( Opts.long "template" <>
-          Opts.short 't' <>
-          Opts.metavar "URL" <>
-          Opts.help "Used during 'update' when building URL. Occurrences of <foo> are replaced with attribute 'foo'."
-        )) <|>
-      ((("type",) . Aeson.String) <$> Opts.strOption
-        ( Opts.long "type" <>
-          Opts.short 'T' <>
-          Opts.metavar "TYPE" <>
-          Opts.help "The type of the URL target. The value can be either 'file' or 'tarball'. If not set, the value is inferred from the suffix of the URL."
-        ))
-
-    parseKeyValJSON = parseKeyVal $ \x ->
-      fromMaybe (Aeson.toJSON x) (Aeson.decodeStrict (B8.pack x))
-
-    -- Parse "key=val" into ("key", val)
-    parseKeyVal
-      :: (String -> Aeson.Value)  -- ^ how to convert to JSON
-      -> String -> Maybe (T.Text, Aeson.Value)
-    parseKeyVal toJSON str = case span (/= '=') str of
-      (key, '=':val) -> Just (T.pack key, toJSON val)
-      _ -> Nothing
-
-    -- Shortcuts for common attributes
-    shortcutAttributes :: Opts.Parser (T.Text, Aeson.Value)
-    shortcutAttributes = foldr (<|>) empty $ mkShortcutAttribute <$>
-      [ "branch", "owner", "repo", "version" ]
-
-    -- TODO: infer those shortcuts from 'Update' keys
-    mkShortcutAttribute :: T.Text -> Opts.Parser (T.Text, Aeson.Value)
-    mkShortcutAttribute = \case
-      attr@(T.uncons -> Just (c,_)) -> fmap (second Aeson.String) $ (attr,) <$> Opts.strOption
-        ( Opts.long (T.unpack attr) <>
-          Opts.short c <>
-          Opts.metavar (T.unpack $ T.toUpper attr) <>
-          Opts.help
-            ( T.unpack $
-              "Equivalent to --attribute " <>
-              attr <> "=<" <> (T.toUpper attr) <> ">"
-            )
-        )
-      _ -> empty
-
 parsePackage :: Opts.Parser (PackageName, PackageSpec)
-parsePackage = (,) <$> parsePackageName <*> parsePackageSpec
+parsePackage = (,) <$> parsePackageName <*> (parsePackageSpec githubCmd)
 
 -------------------------------------------------------------------------------
 -- INIT
@@ -171,14 +105,14 @@ cmdInit = do
               createFile path initNixSourcesJsonContent
               -- Imports @niv@ and @nixpkgs@ (19.03)
               say "Importing 'niv' ..."
-              cmdAdd githubUpdate' (PackageName "niv")
+              cmdAdd (updateCmd githubCmd) (PackageName "niv")
                 (specToFreeAttrs $ PackageSpec $ HMS.fromList
                   [ "owner" .= ("nmattia" :: T.Text)
                   , "repo" .= ("niv" :: T.Text)
                   ]
                 )
               say "Importing 'nixpkgs' ..."
-              cmdAdd githubUpdate' (PackageName "nixpkgs")
+              cmdAdd (updateCmd githubCmd) (PackageName "nixpkgs")
                 (specToFreeAttrs $ PackageSpec $ HMS.fromList
                   [ "owner" .= ("NixOS" :: T.Text)
                   , "repo" .= ("nixpkgs-channels" :: T.Text)
@@ -206,46 +140,73 @@ cmdInit = do
 parseCmdAdd :: Opts.ParserInfo (IO ())
 parseCmdAdd =
     Opts.info
-      ((uncurry (cmdAdd githubUpdate') <$> parseArgs) <**> Opts.helper) $
-      mconcat desc
+      ((parseCommands <|> parseShortcuts) <**> Opts.helper) $
+      (description githubCmd)
   where
-    parseArgs :: Opts.Parser (PackageName, Attrs)
-    parseArgs = collapse <$> parseNameAndGHShortcut <*> parsePackageSpec
-    parseNameAndGHShortcut = (,) <$> optName <*> parseGitHubShortcut
-    -- collaspe a "name or shortcut" with package spec
-    collapse nameAndSpec pspec = (pname, specToLockedAttrs $ pspec <> repoAndOwner)
+    -- XXX: this should parse many shortcuts (github, git). Right now we only
+    -- parse GitHub because the git interface is still experimental.  note to
+    -- implementer: it'll be tricky to have the correct arguments show up
+    -- without repeating "PACKAGE PACKAGE PACKAGE" for every package type.
+    parseShortcuts = parseShortcut githubCmd
+    parseShortcut cmd = uncurry (cmdAdd (updateCmd cmd)) <$> (parseShortcutArgs cmd)
+    parseCmd cmd = uncurry (cmdAdd (updateCmd cmd)) <$> (parseCmdArgs cmd)
+    parseCmdAddGit =
+      Opts.info (parseCmd gitCmd <**> Opts.helper) (description gitCmd)
+    parseCmdAddGitHub =
+      Opts.info (parseCmd githubCmd <**> Opts.helper) (description githubCmd)
+    parseCommands = Opts.subparser
+        ( Opts.hidden <>
+          Opts.commandGroup "Experimental commands:" <>
+          Opts.command "git" parseCmdAddGit <>
+          Opts.command "github" parseCmdAddGitHub
+        )
+
+-- | only used in shortcuts (niv add foo/bar ...) because PACKAGE is NOT
+-- optional
+parseShortcutArgs :: Cmd -> Opts.Parser (PackageName, Attrs)
+parseShortcutArgs cmd = collapse <$> parseNameAndShortcut <*> parsePackageSpec cmd
+  where
+    collapse specAndName pspec = (pname, specToLockedAttrs $ pspec <> baseSpec)
       where
-        (pname, repoAndOwner) = case nameAndSpec of
-          (Just pname', (_, spec)) -> (pname', PackageSpec spec)
-          (Nothing, (pname', spec)) -> (pname', PackageSpec spec)
+        (pname, baseSpec) = case specAndName of
+          ((_, spec), Just pname') -> (pname', PackageSpec spec)
+          ((pname', spec), Nothing) -> (pname', PackageSpec spec)
+    parseNameAndShortcut =
+      (,) <$>
+        Opts.argument
+          (Opts.maybeReader (parseCmdShortcut cmd . T.pack))
+          (Opts.metavar "PACKAGE") <*>
+        optName
     optName = Opts.optional $ PackageName <$> Opts.strOption
       ( Opts.long "name" <>
         Opts.short 'n' <>
         Opts.metavar "NAME" <>
         Opts.help "Set the package name to <NAME>"
       )
 
-    -- parse a github shortcut of the form "owner/repo"
-    parseGitHubShortcut = Opts.strArgument (Opts.metavar "PACKAGE") <&>
-          -- parses a string "owner/repo" into package name (repo) and spec (owner +
-          -- repo)
-          \(T.pack -> str) ->
-            case T.span (/= '/') str of
-              (owner@(T.null -> False)
-                , T.uncons -> Just ('/', repo@(T.null -> False))) ->
-                  ( PackageName repo
-                  , HMS.fromList [ "owner" .= owner, "repo" .= repo ])
-              _ -> (PackageName str, HMS.empty)
-    desc =
-      [ Opts.fullDesc
-      , Opts.progDesc "Add dependency"
-      , Opts.headerDoc $ Just $
-          "Examples:" Opts.<$$>
-          "" Opts.<$$>
-          "  niv add stedolan/jq" Opts.<$$>
-          "  niv add NixOS/nixpkgs-channels -n nixpkgs -b nixos-19.03" Opts.<$$>
-          "  niv add my-package -v alpha-0.1 -t http://example.com/archive/<version>.zip"
-      ]
+-- | only used in command (niv add <cmd> ...) because PACKAGE is optional
+parseCmdArgs :: Cmd -> Opts.Parser (PackageName, Attrs)
+parseCmdArgs cmd = collapse <$> parseNameAndShortcut <*> parsePackageSpec cmd
+  where
+    collapse specAndName pspec = (pname, specToLockedAttrs $ pspec <> baseSpec)
+      where
+        (pname, baseSpec) = case specAndName of
+          (Just (_, spec), Just pname') -> (pname', PackageSpec spec)
+          (Just (pname', spec), Nothing) -> (pname', PackageSpec spec)
+          (Nothing, Just pname') -> (pname', PackageSpec HMS.empty)
+          (Nothing, Nothing) -> (PackageName "unnamed", PackageSpec HMS.empty)
+    parseNameAndShortcut =
+      (,) <$>
+        Opts.optional (Opts.argument
+          (Opts.maybeReader (parseCmdShortcut cmd . T.pack))
+          (Opts.metavar "PACKAGE")) <*>
+        optName
+    optName = Opts.optional $ PackageName <$> Opts.strOption
+      ( Opts.long "name" <>
+        Opts.short 'n' <>
+        Opts.metavar "NAME" <>
+        Opts.help "Set the package name to <NAME>"
+      )
 
 cmdAdd :: Update () a -> PackageName -> Attrs -> IO ()
 cmdAdd updateFunc packageName attrs = do
@@ -297,7 +258,6 @@ showPackage (PackageName pname) (PackageSpec spec) = do
             _ -> tfaint "<barabajagal>"
       tsay $ "  " <> attrName <> ": " <> attrValue
 
-
 -------------------------------------------------------------------------------
 -- UPDATE
 -------------------------------------------------------------------------------
@@ -337,7 +297,7 @@ cmdUpdate = \case
           Just defaultSpec -> do
             fmap attrsToSpec <$> tryEvalUpdate
               (specToLockedAttrs cliSpec <> specToFreeAttrs defaultSpec)
-              (githubUpdate nixPrefetchURL githubLatestRev githubRepo)
+              (updateCmd githubCmd)
 
           Nothing -> abortCannotUpdateNoSuchPackage packageName
 
@@ -356,7 +316,7 @@ cmdUpdate = \case
           let initialSpec = specToFreeAttrs defaultSpec
           finalSpec <- fmap attrsToSpec <$> tryEvalUpdate
             initialSpec
-            (githubUpdate nixPrefetchURL githubLatestRev githubRepo)
+            (updateCmd githubCmd)
           pure finalSpec
 
       let (failed, sources') = partitionEithersHMS esources'
@@ -455,20 +415,6 @@ cmdDrop packageName = \case
       setSources $ Sources $
         HMS.insert packageName packageSpec sources
 
--------------------------------------------------------------------------------
--- Aux
--------------------------------------------------------------------------------
-
-nixPrefetchURL :: Bool -> T.Text -> IO T.Text
-nixPrefetchURL unpack (T.unpack -> url) = do
-    (exitCode, sout, serr) <- runNixPrefetch
-    case (exitCode, lines sout) of
-      (ExitSuccess, l:_)  -> pure $ T.pack l
-      _ -> abortNixPrefetchExpectedOutput (T.pack sout) (T.pack serr)
-  where
-    args = if unpack then ["--unpack", url] else [url]
-    runNixPrefetch = readProcessWithExitCode "nix-prefetch-url" args ""
-
 -------------------------------------------------------------------------------
 -- Files and their content
 -------------------------------------------------------------------------------
@@ -490,10 +436,6 @@ shouldUpdateNixSourcesNix content =
           _ -> False
         _ -> False
 
--- | The IO (real) github update
-githubUpdate' :: Update () ()
-githubUpdate' = githubUpdate nixPrefetchURL githubLatestRev githubRepo
-
 -------------------------------------------------------------------------------
 -- Abort
 -------------------------------------------------------------------------------
@@ -559,12 +501,3 @@ abortUpdateFailed errs = abort $ T.unlines $
       pname <> ": " <> tshow e
     ) errs
 
-abortNixPrefetchExpectedOutput :: T.Text -> T.Text -> IO a
-abortNixPrefetchExpectedOutput sout serr = abort $ [s|
-Could not read the output of 'nix-prefetch-url'. This is a bug. Please create a
-ticket:
-
-  https://github.com/nmattia/niv/issues/new
-
-Thanks! I'll buy you a beer.
-|] <> T.unlines ["stdout: ", sout, "stderr: ", serr]