Skip to content

Commit

Permalink
Support relocated store...
Browse files Browse the repository at this point in the history
... when configured with the NIX_STORE_DIR and NIX_REMOTE
environment variables.
  • Loading branch information
roberth committed Mar 1, 2024
1 parent 872cac9 commit 41a6ce9
Show file tree
Hide file tree
Showing 7 changed files with 140 additions and 40 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
Unreleased

* [Support relocated Nix store](https://github.com/Gabriella439/nix-diff/pull/82)
* If configured through `NIX_REMOTE` environment variable.

1.0.20

* [Bump upper bounds](https://github.com/Gabriella439/nix-diff/pull/79)
Expand Down
3 changes: 2 additions & 1 deletion app/Main.hs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import qualified Data.ByteString.Lazy.Char8
import qualified Data.Text.IO as Text.IO

import Nix.Diff
import Nix.Diff.Store (StorePath(StorePath))
import Nix.Diff.Types
import Nix.Diff.Render.HumanReadable
import Nix.Diff.Transformations
Expand Down Expand Up @@ -174,7 +175,7 @@ main = do
let diffContext = DiffContext {..}
let renderContext = RenderContext {..}
let status = Status Data.Set.empty
let action = diff True left (Data.Set.singleton "out") right (Data.Set.singleton "out")
let action = diff True (StorePath left) (Data.Set.singleton "out") (StorePath right) (Data.Set.singleton "out")
diffTree <- Control.Monad.State.evalStateT (Control.Monad.Reader.runReaderT (unDiff action) diffContext) status
let diffTree' =
transformDiff transformOptions diffTree
Expand Down
1 change: 1 addition & 0 deletions nix-diff.cabal
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ extra-source-files: README.md
library
exposed-modules:
Nix.Diff
Nix.Diff.Store
Nix.Diff.Types
Nix.Diff.Transformations
Nix.Diff.Render.HumanReadable
Expand Down
71 changes: 43 additions & 28 deletions src/Nix/Diff.hs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import Control.Monad (forM)
import Control.Monad.IO.Class (MonadIO, liftIO)
import Control.Monad.Reader (MonadReader, ReaderT, ask)
import Control.Monad.State (MonadState, StateT, get, put)
import Data.Attoparsec.Text (IResult(..))
import Data.Attoparsec.Text (IResult(..), Parser)
import Data.List.NonEmpty (NonEmpty(..))
import Data.Map (Map)
import Data.Maybe (catMaybes)
Expand All @@ -37,7 +37,6 @@ import qualified Data.Text.Encoding.Error
import qualified Data.Vector
import qualified Nix.Derivation
import qualified Patience
import qualified System.Directory as Directory
import qualified System.FilePath as FilePath
import qualified System.Process as Process

Expand All @@ -46,13 +45,15 @@ import Control.Monad.Fail (MonadFail)
#endif

import Nix.Diff.Types
import Nix.Diff.Store (StorePath (StorePath, unsafeStorePathFile))
import qualified Nix.Diff.Store as Store

newtype Status = Status { visited :: Set Diffed }

data Diffed = Diffed
{ leftDerivation :: FilePath
{ leftDerivation :: StorePath
, leftOutput :: Set Text
, rightDerivation :: FilePath
, rightDerivation :: StorePath
, rightOutput :: Set Text
} deriving (Eq, Ord)

Expand Down Expand Up @@ -88,11 +89,11 @@ data Orientation = Character | Word | Line
Nix technically does not require that the Nix store is actually stored
underneath `/nix/store`, but this is the overwhelmingly common use case
-}
derivationName :: FilePath -> Text
derivationName = Text.dropEnd 4 . Text.drop 44 . Text.pack
derivationName :: StorePath -> Text
derivationName = Text.dropEnd 4 . Text.drop 44 . Text.pack . unsafeStorePathFile

-- | Group paths by their name
groupByName :: Map FilePath a -> Map Text (Map FilePath a)
groupByName :: Map StorePath a -> Map Text (Map StorePath a)
groupByName m = Data.Map.fromList assocs
where
toAssoc key = (derivationName key, Data.Map.filterWithKey predicate m)
Expand All @@ -107,11 +108,11 @@ groupByName m = Data.Map.fromList assocs
> /nix/store/${32_CHARACTER_HASH}-${NAME}.drv
-}
buildProductName :: FilePath -> Text
buildProductName = Text.drop 44 . Text.pack
buildProductName :: StorePath -> Text
buildProductName = Text.drop 44 . Text.pack . unsafeStorePathFile

-- | Like `groupByName`, but for `Set`s
groupSetsByName :: Set FilePath -> Map Text (Set FilePath)
groupSetsByName :: Set StorePath -> Map Text (Set StorePath)
groupSetsByName s = Data.Map.fromList (fmap toAssoc (Data.Set.toList s))
where
toAssoc key = (buildProductName key, Data.Set.filter predicate s)
Expand All @@ -127,12 +128,26 @@ readFileUtf8Lenient file =
Data.Text.Encoding.decodeUtf8With Data.Text.Encoding.Error.lenientDecode
<$> Data.ByteString.readFile file

-- TODO: expose in nix-derivation
filepathParser :: Parser FilePath
filepathParser = do
text <- Nix.Derivation.textParser
let str = Text.unpack text
case (Text.uncons text, FilePath.isValid str) of
(Just ('/', _), True) -> do
return str
_ -> do
fail ("bad path ‘" <> Text.unpack text <> "’ in derivation")


-- | Read and parse a derivation from a file
readDerivation :: FilePath -> Diff (Derivation FilePath Text)
readDerivation path = do
readDerivation :: StorePath -> Diff (Derivation StorePath Text)
readDerivation sp = do
path <- liftIO (Store.toPhysicalPath sp)
let string = path
text <- liftIO (readFileUtf8Lenient string)
case Data.Attoparsec.Text.parse Nix.Derivation.parseDerivation text of
let parser = Nix.Derivation.parseDerivationWith (StorePath <$> filepathParser) Nix.Derivation.textParser
case Data.Attoparsec.Text.parse parser text of
Done _ derivation -> do
return derivation
_ -> do
Expand All @@ -141,19 +156,19 @@ readDerivation path = do
-- | Read and parse a derivation from a store path that can be a derivation
-- (.drv) or a realized path, in which case the corresponding derivation is
-- queried.
readInput :: FilePath -> Diff (Derivation FilePath Text)
readInput :: StorePath -> Diff (Derivation StorePath Text)
readInput pathAndMaybeOutput = do
let (path, _) = List.break (== '!') pathAndMaybeOutput
let (path, _) = List.break (== '!') (Store.unsafeStorePathFile pathAndMaybeOutput)
if FilePath.isExtensionOf ".drv" path
then readDerivation path
then readDerivation (StorePath path)
else do
let string = path
result <- liftIO (Process.readProcess "nix-store" [ "--query", "--deriver", string ] [])
case String.lines result of
[] -> fail ("Could not obtain the derivation of " ++ string)
l : ls -> do
let drv_path = Data.List.NonEmpty.last (l :| ls)
readDerivation drv_path
readDerivation (StorePath drv_path)

{-| Join two `Map`s on shared keys, discarding keys which are not present in
both `Map`s
Expand Down Expand Up @@ -206,9 +221,9 @@ getGroupedDiff oldList newList = go $ Patience.diff oldList newList
diffOutput
:: Text
-- ^ Output name
-> (DerivationOutput FilePath Text)
-> (DerivationOutput StorePath Text)
-- ^ Left derivation outputs
-> (DerivationOutput FilePath Text)
-> (DerivationOutput StorePath Text)
-- ^ Right derivation outputs
-> (Maybe OutputDiff)
diffOutput outputName leftOutput rightOutput = do
Expand All @@ -223,9 +238,9 @@ diffOutput outputName leftOutput rightOutput = do

-- | Diff two sets of outputs
diffOutputs
:: Map Text (DerivationOutput FilePath Text)
:: Map Text (DerivationOutput StorePath Text)
-- ^ Left derivation outputs
-> Map Text (DerivationOutput FilePath Text)
-> Map Text (DerivationOutput StorePath Text)
-- ^ Right derivation outputs
-> OutputsDiff
diffOutputs leftOutputs rightOutputs = do
Expand Down Expand Up @@ -363,9 +378,9 @@ diffEnv leftOutputs rightOutputs leftEnv rightEnv = do

-- | Diff input sources
diffSrcs
:: Set FilePath
:: Set StorePath
-- ^ Left input sources
-> Set FilePath
-> Set StorePath
-- ^ Right inputSources
-> Diff SourcesDiff
diffSrcs leftSrcs rightSrcs = do
Expand All @@ -390,12 +405,12 @@ diffSrcs leftSrcs rightSrcs = do
case (Data.Set.toList leftExtraPaths, Data.Set.toList rightExtraPaths) of
([], []) -> return Nothing
([leftPath], [rightPath]) -> do
leftExists <- liftIO (Directory.doesFileExist leftPath)
rightExists <- liftIO (Directory.doesFileExist rightPath)
leftExists <- liftIO (Store.doesFileExist leftPath)
rightExists <- liftIO (Store.doesFileExist rightPath)
srcContentDiff <- if leftExists && rightExists
then do
leftText <- liftIO (readFileUtf8Lenient leftPath)
rightText <- liftIO (readFileUtf8Lenient rightPath)
leftText <- liftIO (Store.readFileUtf8Lenient leftPath)
rightText <- liftIO (Store.readFileUtf8Lenient rightPath)

text <- diffText leftText rightText
return (Just text)
Expand Down Expand Up @@ -427,7 +442,7 @@ diffArgs leftArgs rightArgs = fmap ArgumentsDiff do
let rightList = Data.Vector.toList rightArgs
Data.List.NonEmpty.nonEmpty (Patience.diff leftList rightList)

diff :: Bool -> FilePath -> Set Text -> FilePath -> Set Text -> Diff DerivationDiff
diff :: Bool -> StorePath -> Set Text -> StorePath -> Set Text -> Diff DerivationDiff
diff topLevel leftPath leftOutputs rightPath rightOutputs = do
Status { visited } <- get
let diffed = Diffed leftPath leftOutputs rightPath rightOutputs
Expand Down
7 changes: 4 additions & 3 deletions src/Nix/Diff/Render/HumanReadable.hs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import Control.Monad.Fail (MonadFail)

import Nix.Diff
import Nix.Diff.Types
import qualified Nix.Diff.Store as Store


data RenderContext = RenderContext
Expand Down Expand Up @@ -159,7 +160,7 @@ renderDiffHumanReadable = \case
where
renderOutputStructure os =
renderWith os \(sign, (OutputStructure path outputs)) -> do
echo (sign (Text.pack path <> renderOutputs outputs))
echo (sign (Store.toText path <> renderOutputs outputs))

renderOutputsDiff OutputsDiff{..} = do
ifExist extraOutputs \eo -> do
Expand Down Expand Up @@ -217,7 +218,7 @@ renderDiffHumanReadable = \case
echo (explain ("The input sources named `" <> srcName <> "` differ"))
renderWith srcFileDiff \(sign, paths) -> do
forM_ paths \path -> do
echo (" " <> sign (Text.pack path))
echo (" " <> sign (Store.toText path))

renderInputsDiff InputsDiff{..} = do
renderInputExtraNames inputExtraNames
Expand All @@ -237,7 +238,7 @@ renderDiffHumanReadable = \case
echo (explain ("The set of input derivations named `" <> drvName <> "` do not match"))
renderWith extraPartsDiff \(sign, extraPaths) -> do
forM_ (Data.Map.toList extraPaths) \(extraPath, outputs) -> do
echo (" " <> sign (Text.pack extraPath <> renderOutputs outputs))
echo (" " <> sign (Store.toText extraPath <> renderOutputs outputs))

renderEnvDiff Nothing =
echo (explain "Skipping environment comparison")
Expand Down
76 changes: 76 additions & 0 deletions src/Nix/Diff/Store.hs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
{-# LANGUAGE DeriveDataTypeable #-}
{-# LANGUAGE DerivingStrategies #-}
{-# LANGUAGE GeneralisedNewtypeDeriving #-}
{-# LANGUAGE UndecidableInstances #-}
{-# LANGUAGE NoImportQualifiedPost #-}

-- | A crude implementation of the Nix store concept.
--
-- For anything fancier than this, it would be best to use FFI bindings instead,
-- such as hercules-ci-cnix-store.
module Nix.Diff.Store
( StorePath (..),
toPhysicalPath,
toText,
doesFileExist,
readFileUtf8Lenient,
)
where

import Control.Monad ((<=<))
import Data.Aeson (FromJSON, FromJSONKey, ToJSON, ToJSONKey)
import qualified Data.ByteString
import Data.Data (Data)
import Data.Functor ((<&>))
import qualified Data.List as L
import Data.Text (Text)
import qualified Data.Text
import qualified Data.Text.Encoding
import qualified Data.Text.Encoding.Error
import qualified System.Directory as Directory
import System.Environment (lookupEnv)
import Test.QuickCheck (Arbitrary)

-- | A file path that may not exist on the true file system;
-- needs to be looked up in a store, which may be relocated.
--
-- Unlike the (C++) Nix StorePath type, subpaths are allowed.
newtype StorePath = StorePath
{ -- | If the store is relocated, its physical location is elsewhere, and this 'FilePath' won't resolve.
-- Use 'toPhysicalPath'.
unsafeStorePathFile :: FilePath
}
deriving (Data)
deriving newtype (Show, Eq, Ord, ToJSON, FromJSON, ToJSONKey, FromJSONKey, Arbitrary)

doesFileExist :: StorePath -> IO Bool
doesFileExist =
Directory.doesFileExist <=< toPhysicalPath

readFileUtf8Lenient :: StorePath -> IO Text
readFileUtf8Lenient sp = do
file <- toPhysicalPath sp
Data.Text.Encoding.decodeUtf8With Data.Text.Encoding.Error.lenientDecode
<$> Data.ByteString.readFile file

toPhysicalPath :: StorePath -> IO FilePath
toPhysicalPath (StorePath p) = do
nixStoreDir <- lookupEnv "NIX_STORE_DIR" <&> maybe "/nix/store" stripSlash
nixRemoteMaybe <- lookupEnv "NIX_REMOTE" <&> fmap stripSlash
case nixRemoteMaybe of
Just nixRemote@('/':_) ->
pure $ replaceStart nixStoreDir (nixRemote <> "/" <> nixStoreDir) p
_ -> pure p

-- | Convert a 'StorePath' to a 'Text' for display purposes. The path may not exist at this physical location.
toText :: StorePath -> Text
toText (StorePath p) = Data.Text.pack p

stripSlash :: FilePath -> FilePath
stripSlash = reverse . dropWhile (== '/') . reverse

replaceStart :: String -> String -> String -> String
replaceStart pattern replacement text =
case L.stripPrefix pattern text of
Just rest -> replacement <> rest
Nothing -> text
Loading

0 comments on commit 41a6ce9

Please sign in to comment.