Add support for loading libraries from toplevel

* Start work on loading dune libraries from toplevel
* Finish draft version of toplevel-init-file
* Add toplevel loading script
* Add install stanza for dune.top
* Remove dependency on Unix in the toplevel script
* Add documentation for toplevel integration
* Add a very basic test for toplevel-init-file

Signed-off-by: Marek Bernat <[email protected]>

bin/dune

@@ -23,6 +23,17 @@
  (files
   (dune.exe as dune)))
 
+(install
+ (section lib_root)
+ (package dune)
+ (files
+  (dune.top as ocaml/dune.top)))
+
+(install
+ (section toplevel)
+ (package dune)
+ (files dune.top))
+
 (deprecated_library_name
  (old_public_name dune.configurator)
  (new_public_name dune-configurator))

bin/dune.top

@@ -0,0 +1,26 @@
+let exception AbnormalExit of string * int in
+let run_and_get_lines cmd tmp_filename =
+  let err_code = Sys.command (cmd ^ " > " ^ tmp_filename) in
+  if err_code != 0 then
+    raise (AbnormalExit (cmd, err_code));
+  let inp = open_in tmp_filename in
+  let res = ref [] in
+  let () = try
+    while true do
+      let line = input_line inp in
+      res := List.cons line !res
+    done
+  with End_of_file ->
+    close_in inp
+  in
+  List.rev(!res)
+in
+let exec s =
+  let l = Lexing.from_string s in
+  let ph = !Toploop.parse_toplevel_phrase l in
+  let fmt = Format.make_formatter (fun _ _ _ -> ()) (fun _ -> ()) in
+  let _ = Toploop.execute_phrase false fmt ph in ()
+in
+let tmp_filename = Filename.temp_file "libraries" "top" in
+let lines = run_and_get_lines "dune.exe toplevel-init-file" tmp_filename in
+List.iter (fun l -> exec l) lines

bin/main.ml

@@ -159,6 +159,7 @@ let all =
   ; Compute.command
   ; Upgrade.command
   ; Cache_daemon.command
+  ; Toplevel_init_file.command
   ]
 
 let default =

bin/toplevel_init_file.ml

@@ -0,0 +1,58 @@
+open Stdune
+open Import
+
+let doc =
+  "Print a list of toplevel directives for including directories and loading \
+   cma files."
+
+let man =
+  [ `S "DESCRIPTION"
+  ; `P
+      {|Print a list of toplevel directives for including directories and loading cma files.|}
+  ; `P
+      {|The output of $(b,dune toplevel-init-file) should be evaluated in a toplevel
+          to make a library available there.|}
+  ; `Blocks Common.help_secs
+  ]
+
+let info = Term.info "toplevel-init-file" ~doc ~man
+
+let link_deps link ~lib_config =
+  List.concat_map link ~f:(fun t ->
+      Dune.Lib.link_deps t Dune.Link_mode.Byte lib_config)
+
+let term =
+  let+ common = Common.term
+  and+ dir = Arg.(value & pos 0 string "" & Arg.info [] ~docv:"DIR")
+  and+ ctx_name =
+    Common.context_arg ~doc:{|Select context where to build/run utop.|}
+  in
+  Common.set_common common ~targets:[];
+  Scheduler.go ~common (fun () ->
+      let open Fiber.O in
+      let* setup = Import.Main.setup common in
+      let sctx =
+        Dune.Context_name.Map.find setup.scontexts ctx_name |> Option.value_exn
+      in
+      let dir =
+        Path.Build.relative
+          (Super_context.build_dir sctx)
+          (Common.prefix_target common dir)
+      in
+      let scope = Super_context.find_scope_by_dir sctx dir in
+      let db = Dune.Scope.libs scope in
+      let libs = Dune.Utop.libs_under_dir sctx ~db ~dir:(Path.build dir) in
+      let requires = Dune.Lib.closure ~linking:true libs |> Result.ok_exn in
+      let include_paths = Dune.Lib.L.include_paths requires in
+      let lib_config = sctx |> Super_context.context |> Context.lib_config in
+      let files = link_deps requires ~lib_config in
+      let* () = do_build (List.map files ~f:(fun f -> Target.File f)) in
+      let files_to_load =
+        List.filter files ~f:(fun p ->
+            let ext = Path.extension p in
+            ext = Dune.Mode.compiled_lib_ext Byte || ext = Dune.Cm_kind.ext Cmo)
+      in
+      Dune.Toplevel.print_toplevel_init_file ~include_paths ~files_to_load;
+      Fiber.return ())
+
+let command = (term, info)

doc/dune.inc

@@ -143,6 +143,15 @@
  (package dune)
  (files   dune-subst.1))
 
+(rule
+ (with-stdout-to dune-toplevel-init-file.1
+  (run dune toplevel-init-file --help=groff)))
+
+(install
+ (section man)
+ (package dune)
+ (files   dune-toplevel-init-file.1))
+
 (rule
  (with-stdout-to dune-uninstall.1
   (run dune uninstall --help=groff)))

doc/index.rst

@@ -29,3 +29,4 @@ Welcome to dune's documentation!
    known-issues
    migration
    caching
+   toplevel-integration

doc/toplevel-integration.rst

@@ -0,0 +1,14 @@
+********************
+Toplevel integration
+********************
+
+It's possible to load dune projects in any toplevel. This is achieved in two stages.
+
+First, `dune toplevel-init-file` builds the project and produces a list of toplevel pragmas
+(#directory and #load). Copying the output of this command to a toplevel lets you
+interact with the project's modules.
+
+Second, to enhance usability, dune also provides a toplevel script, which does the above
+manual work for you. To use it, make sure to have `topfind` available in your toplevel by
+invoking `#use "topfind";;`. Afterwards you can run `#use "dune";;` and your
+modules should be available.

src/dune/context.ml

@@ -407,6 +407,8 @@ let create ~(kind : Kind.t) ~path ~env ~env_nodes ~name ~merlin ~targets
                   (Config.local_install_dir ~context:name)
                   "lib/stublibs"))
         ; extend_var "OCAMLPATH" ~path_sep:ocamlpath_sep local_lib_path
+        ; extend_var "OCAMLTOP_INCLUDE_PATH"
+            (Path.relative local_lib_path "toplevel")
         ; extend_var "OCAMLFIND_IGNORE_DUPS_IN" ~path_sep:ocamlpath_sep
             local_lib_path
         ; extend_var "MANPATH"

src/dune/toplevel.ml

@@ -94,6 +94,13 @@ let setup_rules t =
     (Build.symlink ~src:(Path.build src) ~dst);
   setup_module_rules t
 
+let print_toplevel_init_file ~include_paths ~files_to_load =
+  let includes = Path.Set.to_list include_paths in
+  List.iter includes ~f:(fun p ->
+      print_endline ("#directory \"" ^ Path.to_absolute_filename p ^ "\";;"));
+  List.iter files_to_load ~f:(fun p ->
+      print_endline ("#load \"" ^ Path.to_absolute_filename p ^ "\";;"))
+
 module Stanza = struct
   let setup ~sctx ~dir ~(toplevel : Dune_file.Toplevel.t) =
     let source = Source.of_stanza ~dir ~toplevel in

src/dune/toplevel.mli

@@ -18,6 +18,9 @@ val setup_rules : t -> unit
 
 val make : cctx:Compilation_context.t -> source:Source.t -> t
 
+val print_toplevel_init_file :
+  include_paths:Path.Set.t -> files_to_load:Path.t list -> unit
+
 module Stanza : sig
   val setup :
        sctx:Super_context.t

src/dune/utop.mli

@@ -8,4 +8,6 @@ val utop_exe : string
 
 val is_utop_dir : Path.Build.t -> bool
 
+val libs_under_dir : Super_context.t -> db:Lib.DB.t -> dir:Path.t -> Lib.L.t
+
 val setup : Super_context.t -> dir:Path.Build.t -> unit

test/blackbox-tests/dune.inc

@@ -1712,6 +1712,14 @@
    test-cases/tests-stanza-action-syntax-version
    (progn (run %{exe:cram.exe} -test run.t) (diff? run.t run.t.corrected)))))
 
+(rule
+ (alias toplevel-integration)
+ (deps (package dune) (source_tree test-cases/toplevel-integration))
+ (action
+  (chdir
+   test-cases/toplevel-integration
+   (progn (run %{exe:cram.exe} -test run.t) (diff? run.t run.t.corrected)))))
+
 (rule
  (alias toplevel-stanza)
  (deps (package dune) (source_tree test-cases/toplevel-stanza))
@@ -2174,6 +2182,7 @@
   (alias tests-stanza)
   (alias tests-stanza-action)
   (alias tests-stanza-action-syntax-version)
+  (alias toplevel-integration)
   (alias toplevel-stanza)
   (alias trace-file)
   (alias transitive-deps-mode)
@@ -2386,6 +2395,7 @@
   (alias tests-stanza)
   (alias tests-stanza-action)
   (alias tests-stanza-action-syntax-version)
+  (alias toplevel-integration)
   (alias toplevel-stanza)
   (alias trace-file)
   (alias transitive-deps-mode)

test/blackbox-tests/test-cases/toplevel-integration/run.t

@@ -0,0 +1,52 @@
+Test toplevel-init-file on a tiny project
+----------------------------------------------------
+  $ cat >dune-project <<EOF
+  > (lang dune 2.1)
+  > (name test)
+  > EOF
+  $ cat >dune <<EOF
+  > (library
+  >  (name test)
+  >  (public_name test))
+  > EOF
+  $ touch test.opam
+  $ cat >main.ml <<EOF
+  > let hello () = print_endline "hello"
+  > EOF
+
+  $ dune toplevel-init-file
+  #directory "$TESTCASE_ROOT/_build/default/.test.objs/byte";;
+  #directory "$TESTCASE_ROOT/_build/default/.test.objs/native";;
+  #load "$TESTCASE_ROOT/_build/default/test.cma";;
+
+  $ ocaml -stdin <<EOF
+  > #use "topfind";;
+  > #use "dune.top";;
+  > Test.Main.hello ();;
+  > EOF
+  hello
+
+  $ cat >error.ml <<EOF
+  > let oops () = undefined_function ()
+  > EOF
+
+  $ dune toplevel-init-file
+        ocamlc .test.objs/byte/test__Error.{cmi,cmo,cmt} (exit 2)
+  (cd _build/default && /home/marek/.opam/ocaml-base-compiler/bin/ocamlc.opt -w @1..3@5..28@30..39@43@46..47@49..57@61..62-40 -strict-sequence -strict-formats -short-paths -keep-locs -g -bin-annot -I .test.objs/byte -no-alias-deps -opaque -open Test -o .test.objs/byte/test__Error.cmo -c -impl error.ml)
+  File "error.ml", line 1, characters 14-32:
+  1 | let oops () = undefined_function ()
+                    ^^^^^^^^^^^^^^^^^^
+  Error: Unbound value undefined_function
+  [1]
+
+  $ ocaml -stdin <<EOF
+  > #use "topfind";;
+  > #use "dune.top";;
+  > EOF
+        ocamlc .test.objs/byte/test__Error.{cmi,cmo,cmt} (exit 2)
+  (cd _build/default && /home/marek/.opam/ocaml-base-compiler/bin/ocamlc.opt -w @1..3@5..28@30..39@43@46..47@49..57@61..62-40 -strict-sequence -strict-formats -short-paths -keep-locs -g -bin-annot -I .test.objs/byte -no-alias-deps -opaque -open Test -o .test.objs/byte/test__Error.cmo -c -impl error.ml)
+  File "error.ml", line 1, characters 14-32:
+  1 | let oops () = undefined_function ()
+                    ^^^^^^^^^^^^^^^^^^
+  Error: Unbound value undefined_function
+  Exception: AbnormalExit ("dune.exe toplevel-init-file", 1).