Support Git in x-opam-monorepo-opam-repositories

CHANGES.md

@@ -2,6 +2,9 @@
 
 ### Added
 
+- Add support for specifying remote URLs in `x-opam-monorepo-repositories`
+  (#284, #317, @Leonidas-from-XIV)
+
 ### Changed
 
 ### Deprecated

cli/lock.ml

@@ -222,20 +222,75 @@ let interpret_solver_error ~repositories solver = function
       let verbose = display_verbose_diagnostics (Logs.level ()) in
       Opam_solve.diagnostics_message ~verbose solver d
 
+let dirname_of_fpath fpath =
+  fpath |> Fpath.to_string |> OpamFilename.Dir.of_string
+
+let fetch_git_url ~cache_dir dir url =
+  let open OpamProcess.Job.Op in
+  let cache_dir = dirname_of_fpath cache_dir in
+  let dir = dirname_of_fpath dir in
+  OpamGit.B.pull_url ~cache_dir dir None url @@+ function
+  | Not_available (_always_none, url_string) ->
+      Done
+        (Rresult.R.error_msg
+           (Format.asprintf "Could not fetch URL at %s" url_string))
+  | Result _always_none | Up_to_date _always_none -> (
+      OpamGit.B.revision dir @@| function
+      | None -> Rresult.R.error_msg "Could not determine revision of repo"
+      | Some version -> Ok version)
+
+let git_permanent_url (url : OpamUrl.t) version =
+  { url with hash = Some (OpamPackage.Version.to_string version) }
+
 (** Turns each repository URL into a path to the repository's sources,
     eventually fetching them from the remote. *)
 let make_repository_locally_available url =
+  let open OpamProcess.Job.Op in
   match OpamUrl.local_dir url with
   | Some path when Opam.Url.is_local_filesystem url ->
-      Ok (OpamFilename.Dir.to_string OpamFilename.Op.(path / "packages"))
-  | _ ->
-      (* TODO before release *)
-      Rresult.R.error_msg
-        "Only non git, local filesystem URLs (file://) are supported at the \
-         moment"
+      let packages =
+        OpamFilename.Dir.to_string OpamFilename.Op.(path / "packages")
+      in
+      Done (Ok (packages, url))
+  | _ -> (
+      let tmp_dir = Fpath.(Bos.OS.Dir.default_tmp () / "opam-monorepo") in
+      (* the URL might contain all kind of invalid characters like slashes -> hash *)
+      let repo_dir =
+        url |> OpamUrl.to_string |> Digest.string |> Digest.to_hex
+      in
+      let dir = Fpath.(tmp_dir / "repos" / repo_dir) in
+      let cache_dir = Fpath.(tmp_dir / "cache") in
+      let url_result =
+        match url.backend with
+        | `http when String.equal url.path "opam.ocaml.org" ->
+            (* replace OPAM repo with git url *)
+            OpamUrl.parse ~backend:`git
+              "git+https://github.com/ocaml/opam-repository.git"
+            |> Result.ok
+        | `git -> Ok url
+        | `http | `rsync | #OpamUrl.version_control ->
+            Rresult.R.error_msgf
+              "Only git and local file systems (file://) are supported at the \
+               moment, got %a"
+              Opam.Pp.url url
+      in
+      match url_result with
+      | Error _ as e -> Done e
+      | Ok url -> (
+          match Result.List.map ~f:Bos.OS.Dir.create [ cache_dir; dir ] with
+          | Error (`Msg msg) -> Done (Rresult.R.error_msg msg)
+          | Ok _ -> (
+              fetch_git_url ~cache_dir dir url @@| function
+              | Error (`Msg msg) -> Rresult.R.error_msg msg
+              | Ok version ->
+                  let url = git_permanent_url url version in
+                  let packages = Fpath.(dir / "packages" |> to_string) in
+                  Ok (packages, url))))
 
 let make_repositories_locally_available repositories =
-  Result.List.map ~f:make_repository_locally_available repositories
+  repositories
+  |> OpamProcess.Job.seq_map make_repository_locally_available
+  |> OpamProcess.Job.run |> Result.List.all
 
 let opam_env_from_global_state global_state =
   let vars = global_state.OpamStateTypes.global_variables in
@@ -269,29 +324,43 @@ let calculate_opam ~source_config ~build_only ~allow_jbuilder
               l "Solve using explicit repositories:\n%a"
                 Fmt.(list ~sep:(const char '\n') Opam.Pp.url)
                 repositories);
-          let* local_repo_dirs =
-            make_repositories_locally_available repositories
+          let* local_repos = make_repositories_locally_available repositories in
+          let local_repo_dirs, source_config =
+            let local_repo_dirs, repo_urls = List.unzip local_repos in
+            let repositories =
+              repo_urls |> OpamUrl.Set.of_list |> Option.some
+            in
+            let source_config = { source_config with repositories } in
+            (local_repo_dirs, source_config)
           in
           let opam_env = extract_opam_env ~source_config global_state in
           let solver = Opam_solve.explicit_repos_solver in
-          Opam_solve.calculate ~build_only ~allow_jbuilder
-            ~require_cross_compile ~preferred_versions ~local_opam_files
-            ~target_packages ~opam_provided ~pin_depends ?ocaml_version solver
-            (opam_env, local_repo_dirs)
-          |> Result.map_error ~f:(interpret_solver_error ~repositories solver)
+          let dependency_entries =
+            Opam_solve.calculate ~build_only ~allow_jbuilder
+              ~require_cross_compile ~preferred_versions ~local_opam_files
+              ~target_packages ~opam_provided ~pin_depends ?ocaml_version solver
+              (opam_env, local_repo_dirs)
+            |> Result.map_error ~f:(interpret_solver_error ~repositories solver)
+          in
+          let* dependency_entries = dependency_entries in
+          Ok (dependency_entries, source_config)
       | { repositories = None; _ } ->
           OpamSwitchState.with_ `Lock_none global_state (fun switch_state ->
               Logs.info (fun l ->
                   l "Solve using current opam switch: %s"
                     (OpamSwitch.to_string switch_state.switch));
               let solver = Opam_solve.local_opam_config_solver in
-              Opam_solve.calculate ~build_only ~allow_jbuilder
-                ~require_cross_compile ~preferred_versions ~local_opam_files
-                ~target_packages ~opam_provided ~pin_depends ?ocaml_version
-                solver switch_state
-              |> Result.map_error ~f:(fun err ->
-                     let repositories = current_repos ~switch_state in
-                     interpret_solver_error ~repositories solver err)))
+              let dependency_entries =
+                Opam_solve.calculate ~build_only ~allow_jbuilder
+                  ~require_cross_compile ~preferred_versions ~local_opam_files
+                  ~target_packages ~opam_provided ~pin_depends ?ocaml_version
+                  solver switch_state
+                |> Result.map_error ~f:(fun err ->
+                       let repositories = current_repos ~switch_state in
+                       interpret_solver_error ~repositories solver err)
+              in
+              let* dependency_entries = dependency_entries in
+              Ok (dependency_entries, source_config)))
 
 let select_explicitly_specified ~local_packages ~explicitly_specified =
   List.fold_left
@@ -463,7 +532,7 @@ let run (`Root root) (`Recurse_opam recurse) (`Build_only build_only)
   let* preferred_versions =
     preferred_versions ~minimal_update ~root lockfile_path
   in
-  let* dependency_entries =
+  let* dependency_entries, source_config =
     calculate_opam ~source_config ~build_only ~allow_jbuilder
       ~require_cross_compile ~preferred_versions ~ocaml_version
       ~local_opam_files:opam_files ~target_packages

lib/opam.ml

@@ -56,11 +56,8 @@ let has_cross_compile_tag opam =
   let tags = OpamFile.OPAM.tags opam in
   List.mem ~set:tags "cross-compile"
 
-let pull_tree ~url ~hashes ~dir global_state =
+let pull_tree_with_cache' ~cache_dir ~url ~hashes ~dir =
   let dir_str = Fpath.to_string dir in
-  let cache_dir =
-    OpamRepositoryPath.download_cache global_state.OpamStateTypes.root
-  in
   let label = dir_str in
   (* Opam requires a label for the pull, it's only used for logging *)
   let opam_dir = OpamFilename.Dir.of_string dir_str in
@@ -70,6 +67,16 @@ let pull_tree ~url ~hashes ~dir global_state =
   | Not_available (_, long_msg) ->
       Error (`Msg (Printf.sprintf "Failed to pull %s: %s" label long_msg))
 
+let pull_tree ~url ~hashes ~dir global_state =
+  let cache_dir =
+    OpamRepositoryPath.download_cache global_state.OpamStateTypes.root
+  in
+  pull_tree_with_cache' ~cache_dir ~url ~hashes ~dir
+
+let pull_tree_with_cache ~cache_dir =
+  let cache_dir = cache_dir |> Fpath.to_string |> OpamFilename.Dir.of_string in
+  pull_tree_with_cache' ~cache_dir
+
 module Url = struct
   type t = Git of { repo : string; ref : string option } | Other of string
 

lib/opam.mli

@@ -170,6 +170,16 @@ val pull_tree :
     if sucessful and an error otherwise.
     This benefits from opam's global cache.*)
 
+val pull_tree_with_cache :
+  cache_dir:Fpath.t ->
+  url:OpamUrl.t ->
+  hashes:OpamHash.t list ->
+  dir:Fpath.t ->
+  (unit, [> `Msg of string ]) result OpamProcess.job
+(** Pulls the sources from [url] to [dir] using opam's library. Returns the target directory path
+    if sucessful and an error otherwise.
+    Uses a dedicated path for caching. *)
+
 val local_package_version :
   OpamFile.OPAM.t ->
   explicit_version:OpamTypes.version option ->

stdext/list.ml

@@ -59,3 +59,9 @@ let compare ~compare l l' =
     | [], [] -> 0
   in
   aux l l'
+
+let unzip l =
+  let xs, ys =
+    fold_left ~f:(fun (xs, ys) (x, y) -> (x :: xs, y :: ys)) ~init:([], []) l
+  in
+  (rev xs, rev ys)

stdext/list.mli

@@ -14,3 +14,4 @@ val max_exn : compare:('a -> 'a -> int) -> 'a list -> 'a
     Raises Invalig_argument on empty lists *)
 
 val compare : compare:('a -> 'a -> int) -> 'a list -> 'a list -> int
+val unzip : ('a * 'b) list -> 'a list * 'b list

test/bin/explicit-repo.t/explicit-repo.opam

@@ -0,0 +1,9 @@
+opam-version: "2.0"
+depends: [
+  "dune"
+  "git-dep"
+]
+x-opam-monorepo-opam-repositories: [
+  "file://$OPAM_MONOREPO_CWD/minimal-repo"
+  "git+file://$OPAM_MONOREPO_CWD/git-repository"
+]

test/bin/explicit-repo.t/run.t

@@ -0,0 +1,78 @@
+We want to specify an opam-repository that is a git URL. For reproducibility we
+want the locked repository URL to be fixed to the commit that we have used when
+locking.
+
+We have a project that depends on a package "git-dep".
+
+  $ opam show --just-file --raw -fdepends ./explicit-repo.opam
+  dune, git-dep
+
+We start with a minimal `opam-repository`, but the package "git-dep" is not in
+the minimal repository created locally:
+
+  $ gen-minimal-repo
+  $ ls minimal-repo/packages | grep git-dep
+  [1]
+
+"git-dep" is in a repository that is accessible via git, create this repository:
+
+  $ mkdir -p git-repository/packages/git-dep/git-dep.1.0
+  $ cat >git-repository/packages/git-dep/git-dep.1.0/opam <<EOF
+  > opam-version: "2.0"
+  > depends: [
+  >   "dune"
+  > ]
+  > EOF
+  $ cat >git-repository/repo <<EOF
+  > opam-version: "2.0"
+  > EOF
+  $ (cd git-repository ; git init ; git add -A ; git commit -m "initial commit") > /dev/null 2>&1
+  $ SHORT_HASH1=$(git -C git-repository rev-parse --short HEAD)
+
+To find this package, we need to have the git repository in our
+`x-opam-monorepo-opam-repositories` list:
+
+  $ opam show --just-file --raw -fx-opam-monorepo-opam-repositories ./explicit-repo.opam
+  file://$OPAM_MONOREPO_CWD/minimal-repo, git+file://$OPAM_MONOREPO_CWD/git-repository
+
+Having it in there should make opam-monorepo able to find the package and lock
+successfully and picked up the package:
+
+  $ opam-monorepo lock explicit-repo > /dev/null
+  $ opam show --just-file --raw -fdepends ./explicit-repo.opam.locked | grep git-dep
+  "git-dep" {= "1.0"}
+
+To make the build reproducible, opam-monorepo should have replaced the URL to
+the git repo by an URL to the git repo that features the commit hash of the
+repository at the time of locking, so users of the lock file will check out the
+git repository in the same state as the creator of the lock file.
+
+Therefore the list of repositories in the lockfile should have the a git repo
+that's the same as the previous git-repository path but end with "#$SHORT_HASH"
+
+  $ opam show --just-file --raw -fx-opam-monorepo-opam-repositories ./explicit-repo.opam.locked > locked-repos
+  $ grep -Po ".+(?=$SHORT_HASH1)" locked-repos
+  git+file://$OPAM_MONOREPO_CWD/git-repository#
+
+We also need to make sure that the git cache gets updated when the repository is updated:
+
+  $ mkdir -p git-repository/packages/git-dep/git-dep.2.0
+  $ cat >git-repository/packages/git-dep/git-dep.2.0/opam <<EOF
+  > opam-version: "2.0"
+  > depends: [
+  >   "dune"
+  > ]
+  > EOF
+  > (cd git-repository ; git add -A ; git commit -m "New release") > /dev/null 2>&1
+  $ SHORT_HASH2=$(git -C git-repository rev-parse --short HEAD)
+
+Thus a new version of the package has been released in the git repo. Locking it
+anew should pick this version of the dependency as well as the recording the
+new version of the git opam repository.
+
+  $ opam-monorepo lock explicit-repo > /dev/null
+  $ opam show --just-file --raw -fdepends ./explicit-repo.opam.locked | grep git-dep
+  "git-dep" {= "2.0"}
+  $ opam show --just-file --raw -fx-opam-monorepo-opam-repositories ./explicit-repo.opam.locked > locked-repos
+  $ grep -Po ".+(?=$SHORT_HASH2)" locked-repos
+  git+file://$OPAM_MONOREPO_CWD/git-repository#