diff --git a/CHANGES.md b/CHANGES.md index 78fe76b83d4..d19bbf3ed15 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,6 +45,9 @@ Unreleased extensions will now be usable in the toplevel (#3266, fixes #346, @stephanieyou) +- Add a `(subdir ..)` stanza to allow evaluating stanzas in sub directories. + (#3268, @rgrinberg) + 2.4.0 (06/03/2020) ------------------ diff --git a/doc/dune-files.rst b/doc/dune-files.rst index a05b2d39cd5..d467ca7895a 100644 --- a/doc/dune-files.rst +++ b/doc/dune-files.rst @@ -1501,6 +1501,21 @@ run this toplevel with: of `library`_. Currently, ``action`` and ``future_syntax`` are not supported in the toplevel. +subdir +------ + +The ``subdir`` stanza can be used to evaluate stanzas in sub directories. This is +useful for generated files or to override stanzas in vendored direcotries +without editing vendored dune files. + +In this example, a ``bar`` target is created in the ``foo`` directory, and a bar +target will be created in ``a/b/bar``: + +.. code:: scheme + + (subdir foo (rule (with-stdout-to bar (echo baz)))) + (subdir a/b (rule (with-stdout-to bar (echo baz)))) + external_variant ----------------- diff --git a/src/dune/dir_contents.ml b/src/dune/dir_contents.ml index 87ceb95a3d8..ae3a930bdf2 100644 --- a/src/dune/dir_contents.ml +++ b/src/dune/dir_contents.ml @@ -201,9 +201,10 @@ end = struct | Group_root _ -> acc and walk_children ft_dir ~dir ~local acc = - File_tree.Dir.fold_sub_dirs ft_dir ~init:acc ~f:(fun name ft_dir acc -> - let dir = Path.Build.relative dir name in - let local = name :: local in + File_tree.Dir.fold_sub_dirs ft_dir ~init:acc + ~f:(fun ~basename ft_dir acc -> + let dir = Path.Build.relative dir basename in + let local = basename :: local in walk ft_dir ~dir ~local acc) in walk_children ft_dir ~dir ~local:[] [] diff --git a/src/dune/dune_load.ml b/src/dune/dune_load.ml index 6fcb267711d..11b95049d03 100644 --- a/src/dune/dune_load.ml +++ b/src/dune/dune_load.ml @@ -111,7 +111,10 @@ module Dune_files = struct type one = | Literal of Dune_file.t - | Script of script + | Script of + { script : script + ; from_parent : Dune_lang.Ast.t list + } type t = one list @@ -157,9 +160,9 @@ module Dune_files = struct let static, dynamic = List.partition_map dune_files ~f:(function | Literal x -> Left x - | Script x -> Right x) + | Script { script; from_parent } -> Right (script, from_parent)) in - Fiber.parallel_map dynamic ~f:(fun { dir; file; project } -> + Fiber.parallel_map dynamic ~f:(fun ({ dir; file; project }, from_parent) -> let generated_dune_file = Path.Build.append_source (Path.Build.relative generated_dune_files_dir @@ -191,6 +194,7 @@ module Dune_files = struct ]; Fiber.return ( Dune_lang.Parser.load (Path.build generated_dune_file) ~mode:Many + |> List.rev_append from_parent |> Dune_file.parse ~dir ~file ~project )) >>| fun dynamic -> static @ dynamic end @@ -203,11 +207,13 @@ type conf = let interpret ~dir ~project ~(dune_file : File_tree.Dune_file.t) = let file = File_tree.Dune_file.path dune_file in - match dune_file with - | Ocaml_script _ -> Dune_files.Script { dir; project; file } - | Plain p -> - let sexps = File_tree.Dune_file.Plain.get_sexp_and_destroy p in - Literal (Dune_file.parse sexps ~dir ~file ~project) + let static = + File_tree.Dune_file.get_static_sexp_and_possibly_destroy dune_file + in + match File_tree.Dune_file.kind dune_file with + | Ocaml_script -> + Dune_files.Script { script = { dir; project; file }; from_parent = static } + | Plain -> Literal (Dune_file.parse static ~dir ~file ~project) let load ~ancestor_vcs () = File_tree.init ~ancestor_vcs ~recognize_jbuilder_projects:false; @@ -239,21 +245,12 @@ let load ~ancestor_vcs () = (Path.Source.to_string_maybe_quoted (Package.opam_file b)) ])) in - let rec walk dir dune_files = - if File_tree.Dir.status dir = Data_only then - dune_files - else - let path = File_tree.Dir.path dir in - let project = File_tree.Dir.project dir in - let dune_files = - match File_tree.Dir.dune_file dir with - | None -> dune_files - | Some dune_file -> - let dune_file = interpret ~dir:path ~project ~dune_file in - dune_file :: dune_files - in - File_tree.Dir.fold_sub_dirs dir ~init:dune_files - ~f:(fun _name dir dune_files -> walk dir dune_files) + let dune_files = + File_tree.Dir.fold_dune_files (File_tree.root ()) ~init:[] + ~f:(fun ~basename:_ dir dune_file dune_files -> + let path = File_tree.Dir.path dir in + let project = File_tree.Dir.project dir in + let dune_file = interpret ~dir:path ~project ~dune_file in + dune_file :: dune_files) in - let dune_files = walk (File_tree.root ()) [] in { dune_files; packages; projects } diff --git a/src/dune/file_tree.ml b/src/dune/file_tree.ml index 21bd5ad4505..e8c727459f0 100644 --- a/src/dune/file_tree.ml +++ b/src/dune/file_tree.ml @@ -2,31 +2,29 @@ open! Stdune open Import module File = struct - type t = - { ino : int - ; dev : int - } + module T = struct + type t = + { ino : int + ; dev : int + } - let to_dyn { ino; dev } = - let open Dyn.Encoder in - record [ ("ino", Int.to_dyn ino); ("dev", Int.to_dyn dev) ] + let to_dyn { ino; dev } = + let open Dyn.Encoder in + record [ ("ino", Int.to_dyn ino); ("dev", Int.to_dyn dev) ] + + let compare a b = + match Int.compare a.ino b.ino with + | Eq -> Int.compare a.dev b.dev + | ne -> ne + end - let compare a b = - match Int.compare a.ino b.ino with - | Eq -> Int.compare a.dev b.dev - | ne -> ne + include T let dummy = { ino = 0; dev = 0 } let of_stats (st : Unix.stats) = { ino = st.st_ino; dev = st.st_dev } - module Map = Map.Make (struct - type nonrec t = t - - let compare = compare - - let to_dyn _ = Dyn.opaque - end) + module Map = Map.Make (T) let of_source_path p = of_stats (Path.stat (Path.source p)) end @@ -34,49 +32,80 @@ end module Dune_file = struct module Plain = struct type t = - { path : Path.Source.t - ; sub_dirs : Predicate_lang.Glob.t Sub_dirs.Status.Map.t - ; mutable sexps : Dune_lang.Ast.t list + { mutable contents : Sub_dirs.Dir_map.per_dir + ; for_subdirs : Sub_dirs.Dir_map.t } + (** It's also possible to add GC for: + + - [contents.subdir_status] + - [consumed nodes of for_subdirs] + + We don't do this for now because the benefits are likely small.*) + let get_sexp_and_destroy t = - let sexps = t.sexps in - t.sexps <- []; - sexps + let result = t.contents.sexps in + t.contents <- { t.contents with sexps = [] }; + result end let fname = "dune" let jbuild_fname = "jbuild" + type kind = + | Plain + | Ocaml_script + type t = - | Plain of Plain.t - | Ocaml_script of Path.Source.t - - let sub_dirs = function - | Some (Plain p) -> p.sub_dirs - | None - | Some (Ocaml_script _) -> - Sub_dirs.default - - let path = function - | Plain x -> x.path - | Ocaml_script p -> p - - let load file ~project = - Io.with_lexbuf_from_file (Path.source file) ~f:(fun lb -> - if Dune_lexer.is_script lb then - Ocaml_script file - else - let sexps = Dune_lang.Parser.parse lb ~mode:Many in - let decoder = - Dune_project.set_parsing_context project Sub_dirs.decode - in - let sub_dirs, sexps = - Dune_lang.Decoder.parse decoder Univ_map.empty - (Dune_lang.Ast.List (Loc.none, sexps)) - in - Plain { path = file; sexps; sub_dirs }) + { path : Path.Source.t + ; kind : kind + ; (* for [kind = Ocaml_script], this is the part inserted with subdir *) + plain : Plain.t + } + + let get_static_sexp_and_possibly_destroy t = + match t.kind with + | Ocaml_script -> t.plain.contents.sexps + | Plain -> Plain.get_sexp_and_destroy t.plain + + let kind t = t.kind + + let path t = t.path + + let sub_dirs (t : t option) = + match t with + | None -> Sub_dirs.default + | Some t -> Sub_dirs.or_default t.plain.contents.subdir_status + + let load_plain sexps ~from_parent ~project = + let decoder = Dune_project.set_parsing_context project Sub_dirs.decode in + let active = + let parsed = + Dune_lang.Decoder.parse decoder Univ_map.empty + (Dune_lang.Ast.List (Loc.none, sexps)) + in + match from_parent with + | None -> parsed + | Some from_parent -> Sub_dirs.Dir_map.merge parsed from_parent + in + let contents = Sub_dirs.Dir_map.root active in + { Plain.contents; for_subdirs = active } + + let load file ~file_exists ~from_parent ~project = + let kind, plain = + match file_exists with + | false -> (Plain, load_plain [] ~from_parent ~project) + | true -> + Io.with_lexbuf_from_file (Path.source file) ~f:(fun lb -> + if Dune_lexer.is_script lb then + let from_parent = load_plain [] ~from_parent ~project in + (Ocaml_script, from_parent) + else + let sexps = Dune_lang.Parser.parse lb ~mode:Many in + (Plain, load_plain sexps ~from_parent ~project)) + in + { path = file; kind; plain } end module Readdir : sig @@ -152,19 +181,72 @@ end = struct |> Result.ok end +module Dirs_visited : sig + (** Unique set of all directories visited *) + type t + + val singleton : Path.Source.t -> t + + module Per_fn : sig + (** Stores the directories visited per node (basename) *) + type t + + type dirs_visited + + val to_dyn : t -> Dyn.t + + val init : t + + val find : t -> Path.Source.t -> dirs_visited + + val add : t -> dirs_visited -> string * Path.Source.t * File.t -> t + end + with type dirs_visited := t +end = struct + type t = Path.Source.t File.Map.t + + let singleton path = File.Map.singleton (File.of_source_path path) path + + module Per_fn = struct + type nonrec t = t String.Map.t + + let init = String.Map.empty + + let find t path = + String.Map.find t (Path.Source.basename path) + |> Option.value ~default:File.Map.empty + + let add (acc : t) dirs_visited (fn, path, file) = + if Sys.win32 then + acc + else + let new_dirs_visited = + File.Map.update dirs_visited file ~f:(function + | None -> Some path + | Some first_path -> + User_error.raise + [ Pp.textf + "Path %s has already been scanned. Cannot scan it again \ + through symlink %s" + (Path.Source.to_string_maybe_quoted first_path) + (Path.Source.to_string_maybe_quoted path) + ]) + in + String.Map.add_exn acc fn new_dirs_visited + + let to_dyn t = String.Map.to_dyn (File.Map.to_dyn Path.Source.to_dyn) t + end +end + module Output = struct type 'a t = { dir : 'a - ; visited : Path.Source.t File.Map.t String.Map.t + ; visited : Dirs_visited.Per_fn.t } let to_dyn f { dir; visited } = let open Dyn.Encoder in - record - [ ("dir", f dir) - ; ( "visited" - , String.Map.to_dyn (File.Map.to_dyn Path.Source.to_dyn) visited ) - ] + record [ ("dir", f dir); ("visited", Dirs_visited.Per_fn.to_dyn visited) ] end module Dir0 = struct @@ -184,6 +266,7 @@ module Dir0 = struct and sub_dir = { sub_dir_status : Sub_dirs.Status.t + ; virtual_ : bool ; sub_dir_as_t : ( Path.Source.t , t Output.t option @@ -200,12 +283,13 @@ module Dir0 = struct ; ("vcs", Dyn.Encoder.option Vcs.to_dyn vcs) ] - and dyn_of_sub_dir { sub_dir_status; sub_dir_as_t } = + and dyn_of_sub_dir { sub_dir_status; sub_dir_as_t; virtual_ } = let open Dyn.Encoder in let path = Memo.Cell.input sub_dir_as_t in record [ ("status", Sub_dirs.Status.to_dyn sub_dir_status) ; ("sub_dir_as_t", Path.Source.to_dyn path) + ; ("virtual_", bool virtual_) ] and dyn_of_contents { files; sub_dirs; dune_file } = @@ -306,76 +390,113 @@ module rec Memoized : sig end = struct open Memoized - let get_sub_dirs ~dirs_visited ~dirs ~sub_dirs - ~(dir_status : Sub_dirs.Status.t) = - let sub_dirs = - Sub_dirs.eval sub_dirs ~dirs:(List.map ~f:(fun (a, _, _) -> a) dirs) - in - dirs - |> List.fold_left ~init:(String.Map.empty, String.Map.empty) - ~f:(fun (dirs_visited_acc, subdirs) (fn, path, file) -> - let status = Sub_dirs.status sub_dirs ~dir:fn in - match status with - | Ignored -> (dirs_visited_acc, subdirs) - | Status status -> - let dir_status : Sub_dirs.Status.t = - match (dir_status, status) with - | Data_only, _ -> Data_only - | Vendored, Normal -> Vendored - | _, _ -> status - in - let dirs_visited_acc = - if Sys.win32 then - dirs_visited_acc - else - let new_dirs_visited = - File.Map.update dirs_visited file ~f:(function - | None -> Some path - | Some first_path -> - User_error.raise - [ Pp.textf - "Path %s has already been scanned. Cannot scan it \ - again through symlink %s" - (Path.Source.to_string_maybe_quoted first_path) - (Path.Source.to_string_maybe_quoted path) - ]) - in - String.Map.add_exn dirs_visited_acc fn new_dirs_visited - in - let sub_dir = - let sub_dir_as_t = find_dir_raw path in - { Dir0.sub_dir_status = dir_status; sub_dir_as_t } - in - let subdirs = String.Map.set subdirs fn sub_dir in - (dirs_visited_acc, subdirs)) + module Get_subdir : sig + (** Get all the sub directories of [path].*) + val all : + dirs_visited:Dirs_visited.t + -> dirs:(string * Path.Source.t * File.t) list + -> sub_dirs:Predicate_lang.Glob.t Sub_dirs.Status.Map.t + -> parent_status:Sub_dirs.Status.t + -> dune_file:Dune_file.t option (** to interpret [(subdir ..)] stanzas *) + -> path:Path.Source.t + -> Dirs_visited.Per_fn.t * Dir0.sub_dir String.Map.t + end = struct + let status ~status_map ~(parent_status : Sub_dirs.Status.t) dir : + Sub_dirs.Status.t option = + let status = Sub_dirs.status status_map ~dir in + match status with + | Ignored -> None + | Status status -> + Some + ( match (parent_status, status) with + | Data_only, _ -> Data_only + | Vendored, Normal -> Vendored + | _, _ -> status ) + + let make_subdir ~dir_status ~virtual_ path = + let sub_dir_as_t = find_dir_raw path in + { Dir0.sub_dir_status = dir_status; sub_dir_as_t; virtual_ } + + let physical ~dirs_visited ~dirs ~sub_dirs ~parent_status = + let status_map = + Sub_dirs.eval sub_dirs ~dirs:(List.map ~f:(fun (a, _, _) -> a) dirs) + in + List.fold_left dirs ~init:(Dirs_visited.Per_fn.init, String.Map.empty) + ~f:(fun (dirs_visited_acc, subdirs) ((fn, path, _) as dir) -> + match status ~status_map ~parent_status fn with + | None -> (dirs_visited_acc, subdirs) + | Some dir_status -> + let dirs_visited_acc = + Dirs_visited.Per_fn.add dirs_visited_acc dirs_visited dir + in + let sub_dir = make_subdir ~dir_status ~virtual_:false path in + let subdirs = String.Map.add_exn subdirs fn sub_dir in + (dirs_visited_acc, subdirs)) + + let virtual_ ~sub_dirs ~parent_status ~dune_file ~init ~path = + match dune_file with + | None -> init + | Some (df : Dune_file.t) -> + let dirs = Sub_dirs.Dir_map.sub_dirs df.plain.for_subdirs in + let status_map = Sub_dirs.eval sub_dirs ~dirs in + List.fold_left dirs ~init ~f:(fun acc fn -> + let path = Path.Source.relative path fn in + match status ~status_map ~parent_status fn with + | None -> acc + | Some dir_status -> + String.Map.update acc fn ~f:(function + (* Directories which are physical are skipped *) + | Some _ as r -> r + | None -> Some (make_subdir ~dir_status ~virtual_:true path))) + + let all ~dirs_visited ~dirs ~sub_dirs ~parent_status ~dune_file ~path = + let visited, init = + physical ~dirs_visited ~dirs ~sub_dirs ~parent_status + in + let init = virtual_ ~sub_dirs ~parent_status ~dune_file ~init ~path in + (visited, init) + end let dune_file ~(dir_status : Sub_dirs.Status.t) ~recognize_jbuilder_projects ~path ~files ~project = - if dir_status = Data_only then - None - else if - (not recognize_jbuilder_projects) - && String.Set.mem files Dune_file.jbuild_fname - then - User_error.raise - ~loc: - (Loc.in_file - (Path.source (Path.Source.relative path Dune_file.jbuild_fname))) - [ Pp.text - "jbuild files are no longer supported, please convert this file to \ - a dune file instead." - ; Pp.text - "Note: You can use \"dune upgrade\" to convert your project to \ - dune." - ] - else if not (String.Set.mem files Dune_file.fname) then + let file_exists = + if dir_status = Data_only then + false + else if + (not recognize_jbuilder_projects) + && String.Set.mem files Dune_file.jbuild_fname + then + User_error.raise + ~loc: + (Loc.in_file + (Path.source (Path.Source.relative path Dune_file.jbuild_fname))) + [ Pp.text + "jbuild files are no longer supported, please convert this file \ + to a dune file instead." + ; Pp.text + "Note: You can use \"dune upgrade\" to convert your project to \ + dune." + ] + else + String.Set.mem files Dune_file.fname + in + let from_parent = + let open Option.O in + let* parent = Path.Source.parent path in + let* parent = find_dir parent in + let* dune_file = parent.contents.dune_file in + let dir_basename = Path.Source.basename path in + Sub_dirs.Dir_map.descend dune_file.plain.for_subdirs dir_basename + in + let dune_file_absent = (not file_exists) && Option.is_none from_parent in + if dune_file_absent then None else ( ignore ( Dune_project.ensure_project_file_exists project : Dune_project.created_or_already_exist ); let file = Path.Source.relative path Dune_file.fname in - Some (Dune_file.load file ~project) + Some (Dune_file.load file ~file_exists ~project ~from_parent) ) let contents { Readdir.dirs; files } ~dirs_visited ~project ~path @@ -389,7 +510,8 @@ end = struct in let sub_dirs = Dune_file.sub_dirs dune_file in let dirs_visited, sub_dirs = - get_sub_dirs ~dirs_visited ~dirs ~sub_dirs ~dir_status + Get_subdir.all ~dirs_visited ~dirs ~sub_dirs ~parent_status:dir_status + ~dune_file ~path in (Dir0.Contents.create ~files ~sub_dirs ~dune_file, dirs_visited) @@ -416,7 +538,7 @@ end = struct | Some p -> p in let vcs = settings.ancestor_vcs in - let dirs_visited = File.Map.singleton (File.of_source_path path) path in + let dirs_visited = Dirs_visited.singleton path in let contents, visited = contents readdir ~dirs_visited ~project ~path ~dir_status in @@ -442,20 +564,20 @@ end = struct let* { Output.dir = parent_dir; visited = dirs_visited } = Memo.Cell.get_sync (find_dir_raw parent_dir) in - let* dir_status = + let* dir_status, virtual_ = let basename = Path.Source.basename path in let+ sub_dir = String.Map.find parent_dir.contents.sub_dirs basename in - sub_dir.sub_dir_status - in - let dirs_visited = - String.Map.find dirs_visited (Path.Source.basename path) - |> Option.value ~default:File.Map.empty + (sub_dir.sub_dir_status, sub_dir.virtual_) in + let dirs_visited = Dirs_visited.Per_fn.find dirs_visited path in let settings = Settings.get () in let readdir = - match Readdir.of_source_path path with - | Ok dir -> dir - | Error _ -> Readdir.empty + if virtual_ then + Readdir.empty + else + match Readdir.of_source_path path with + | Ok dir -> dir + | Error _ -> Readdir.empty in let project = if dir_status = Data_only then @@ -538,8 +660,21 @@ module Dir = struct (Memo.Cell.get_sync s.sub_dir_as_t |> Option.value_exn).dir let fold_sub_dirs (t : t) ~init ~f = - String.Map.foldi t.contents.sub_dirs ~init ~f:(fun name s acc -> - f name (sub_dir_as_t s) acc) + String.Map.foldi t.contents.sub_dirs ~init ~f:(fun basename s acc -> + f ~basename (sub_dir_as_t s) acc) + + let fold_dune_files (type acc) t ~(init : acc) ~f = + let rec loop ~basename dir (acc : acc) : acc = + let init = + match dune_file dir with + | None -> acc + | Some dune_file -> f ~basename dir dune_file acc + in + fold_sub_dirs dir ~init ~f:(fun ~basename -> + loop ~basename:(Some basename)) + in + let basename = Path.Source.basename_opt t.path in + loop ~basename t init let rec fold t ~traverse ~init:acc ~f = let must_traverse = Sub_dirs.Status.Map.find traverse t.status in @@ -547,7 +682,7 @@ module Dir = struct | false -> acc | true -> let acc = f t acc in - fold_sub_dirs t ~init:acc ~f:(fun _name t acc -> + fold_sub_dirs t ~init:acc ~f:(fun ~basename:_ t acc -> fold t ~traverse ~init:acc ~f) end diff --git a/src/dune/file_tree.mli b/src/dune/file_tree.mli index 2e03d87d5fd..989a732fdbc 100644 --- a/src/dune/file_tree.mli +++ b/src/dune/file_tree.mli @@ -4,22 +4,22 @@ open! Stdune open! Import module Dune_file : sig - module Plain : sig - (** [sexps] is mutable as we get rid of the S-expressions once they have - been parsed, in order to release the memory as soon as we don't need - them. *) - type t - - val get_sexp_and_destroy : t -> Dune_lang.Ast.t list - end - val fname : string val jbuild_fname : string - type t = private - | Plain of Plain.t - | Ocaml_script of Path.Source.t + type kind = private + | Plain + | Ocaml_script + + type t + + (** We release the memory taken by s-exps as soon as it is used, unless + [kind = Ocaml_script]. In which case that optimization is incorrect as we + need to re-parse in every context. *) + val get_static_sexp_and_possibly_destroy : t -> Dune_lang.Ast.t list + + val kind : t -> kind val path : t -> Path.Source.t end @@ -33,7 +33,13 @@ module Dir : sig val file_paths : t -> Path.Source.Set.t - val fold_sub_dirs : t -> init:'a -> f:(string -> t -> 'a -> 'a) -> 'a + val fold_sub_dirs : t -> init:'a -> f:(basename:string -> t -> 'a -> 'a) -> 'a + + val fold_dune_files : + t + -> init:'acc + -> f:(basename:string option -> t -> Dune_file.t -> 'acc -> 'acc) + -> 'acc val sub_dir_paths : t -> Path.Source.Set.t diff --git a/src/dune/sub_dirs.ml b/src/dune/sub_dirs.ml index 5e0cbedbcfb..0b8c84ab2ca 100644 --- a/src/dune/sub_dirs.ml +++ b/src/dune/sub_dirs.ml @@ -13,6 +13,12 @@ module Status = struct ; normal : 'a } + let merge x y ~f = + { data_only = f x.data_only y.data_only + ; vendored = f x.vendored y.vendored + ; normal = f x.normal y.normal + } + let find { data_only; vendored; normal } = function | Data_only -> data_only | Vendored -> vendored @@ -25,6 +31,9 @@ module Status = struct ; ("vendored", f vendored) ; ("normal", f normal) ] + + let init f = + { data_only = f Data_only; vendored = f Vendored; normal = f Normal } end let to_dyn t = @@ -67,14 +76,23 @@ let default = ; vendored = Predicate_lang.empty } +let or_default (t : _ Status.Map.t) : _ Status.Map.t = + Status.Map.init (fun kind -> + match Status.Map.find t kind with + | None -> Status.Map.find default kind + | Some (_loc, s) -> s) + let make ~dirs ~data_only ~ignored_sub_dirs ~vendored_dirs = - let normal = Option.value dirs ~default:default.normal in let data_only = - let data_only = Option.value data_only ~default:default.data_only in - Predicate_lang.union (data_only :: ignored_sub_dirs) + match (data_only, ignored_sub_dirs) with + | None, [] -> None + | Some (loc, data_only), [] -> Some (loc, data_only) + | None, (loc, _) :: _ -> + let ignored_sub_dirs = List.map ~f:snd ignored_sub_dirs in + Some (loc, Predicate_lang.union ignored_sub_dirs) + | Some _data_only, _ :: _ -> assert false in - let vendored = Option.value vendored_dirs ~default:default.vendored in - { Status.Map.normal; data_only; vendored } + { Status.Map.normal = dirs; data_only; vendored = vendored_dirs } type status_map = Status.t String.Map.t @@ -112,34 +130,106 @@ let eval (t : _ Status.Map.t) ~dirs = dir ])) -let decode = +type subdir_stanzas = (Loc.t * Predicate_lang.Glob.t) option Status.Map.t + +module Dir_map = struct + type per_dir = + { sexps : Dune_lang.Ast.t list + ; subdir_status : subdir_stanzas + } + + type t = + { data : per_dir + ; nodes : t String.Map.t + } + + let empty_per_dir = + { sexps = []; subdir_status = Status.Map.init (fun _ -> None) } + + let empty = { data = empty_per_dir; nodes = String.Map.empty } + + let root t = t.data + + let descend t (p : string) = String.Map.find t.nodes p + + let sub_dirs t = String.Map.keys t.nodes + + let rec make_at_path path data = + match path with + | [] -> data + | x :: xs -> + let nodes = String.Map.singleton x (make_at_path xs data) in + { empty with nodes } + + let singleton data = { empty with data } + + let merge_data d1 d2 = + { sexps = d1.sexps @ d2.sexps + ; subdir_status = + Status.Map.merge d1.subdir_status d2.subdir_status ~f:(fun l r -> + match (l, r) with + | acc, None + | None, acc -> + acc + | Some (loc, _), Some (loc2, _) -> + User_error.raise ~loc + [ Pp.text "This stanza stanza was already specified at:" + ; Pp.verbatim (Loc.to_file_colon_line loc2) + ]) + } + + let rec merge t1 t2 : t = + let data = merge_data t1.data t2.data in + let nodes = + String.Map.union t1.nodes t2.nodes ~f:(fun _ l r -> Some (merge l r)) + in + { data; nodes } + + let merge_all = List.fold_left ~f:merge ~init:empty +end + +let descedant_path = + Dune_lang.Decoder.plain_string (fun ~loc fn -> + Path.Local.parse_string_exn ~loc fn |> Path.Local.explode) + +let strict_subdir field_name = let open Dune_lang.Decoder in - let strict_subdir field_name = - plain_string (fun ~loc dn -> - let msg = [ Pp.textf "invalid sub-directory name %S" dn ] in - if Filename.dirname dn <> Filename.current_dir_name then - let msg = - [ Pp.textf "only immediate sub-directories may be specified." ] - in - let hints = - [ Pp.textf "to ignore %s, write \"(%s %s)\" in %s/dune" dn - field_name (Filename.basename dn) (Filename.dirname dn) - ] - in + plain_string (fun ~loc dn -> + let msg = [ Pp.textf "invalid sub-directory name %S" dn ] in + if Filename.dirname dn <> Filename.current_dir_name then + let msg = + [ Pp.textf "only immediate sub-directories may be specified." ] + in + let hints = + [ Pp.textf "to ignore %s, write \"(%s %s)\" in %s/dune" dn field_name + (Filename.basename dn) (Filename.dirname dn) + ] + in + User_error.raise ~loc ~hints msg + else if + match dn with + | "" + | "." -> + let hints = [ Pp.textf "did you mean (%s *)?" field_name ] in User_error.raise ~loc ~hints msg - else if - match dn with - | "" - | "." -> - let hints = [ Pp.textf "did you mean (%s *)?" field_name ] in - User_error.raise ~loc ~hints msg - | ".." -> true - | _ -> false - then - User_error.raise ~loc msg - else - (loc, dn)) + | ".." -> true + | _ -> false + then + User_error.raise ~loc msg + else + (loc, dn)) + +let strict_subdir_glob field_name = + let open Dune_lang.Decoder in + let+ globs = + repeat + (let+ loc, l = strict_subdir field_name in + Predicate_lang.Glob.of_glob (Glob.of_string_exn loc l)) in + Predicate_lang.union globs + +let decode = + let open Dune_lang.Decoder in let ignored_sub_dirs = let ignored = let+ l = enter (repeat (strict_subdir "ignored_sub_dirs")) in @@ -155,14 +245,6 @@ let decode = ]; ignored in - let strict_subdir_glob field_name = - let+ globs = - repeat - (let+ loc, l = strict_subdir field_name in - Predicate_lang.Glob.of_glob (Glob.of_string_exn loc l)) - in - Predicate_lang.union globs - in let dirs = located ( Dune_lang.Syntax.since Stanza.syntax (1, 6) @@ -179,11 +261,28 @@ let decode = ( Dune_lang.Syntax.since Stanza.syntax (1, 11) >>> strict_subdir_glob "vendored_dirs" ) in - let decode = + let rec subdir () = + let* () = Dune_lang.Syntax.since Stanza.syntax (2, 5) in + let* subdir = descedant_path in + let+ node = fields (decode ~allow_ignored_subdirs:false) in + Dir_map.make_at_path subdir node + and decode ~allow_ignored_subdirs = let+ dirs = field_o "dirs" dirs and+ data_only = field_o "data_only_dirs" data_only_dirs - and+ ignored_sub_dirs = multi_field "ignored_subdirs" ignored_sub_dirs + and+ ignored_sub_dirs = + let parser = + if allow_ignored_subdirs then + ignored_sub_dirs + else + let+ loc = loc in + User_error.raise ~loc + [ Pp.textf + "ignored_subdirs is not allowed under subdir. Use dirs instead" + ] + in + multi_field "ignored_subdirs" (located parser) and+ vendored_dirs = field_o "vendored_dirs" vendored_dirs + and+ subdirs = multi_field "subdir" (subdir ()) and+ rest = leftover_fields in match (data_only, dirs, ignored_sub_dirs) with | None, Some (loc, _), _ :: _ -> @@ -198,9 +297,10 @@ let decode = dune file. " ] | _ -> - let dirs = Option.map ~f:snd dirs in - let data_only = Option.map ~f:snd data_only in - let vendored_dirs = Option.map ~f:snd vendored_dirs in - (make ~dirs ~data_only ~ignored_sub_dirs ~vendored_dirs, rest) + Dir_map.merge_all + (let subdir_status = + make ~dirs ~data_only ~ignored_sub_dirs ~vendored_dirs + in + Dir_map.singleton { Dir_map.sexps = rest; subdir_status } :: subdirs) in - enter (fields decode) + enter (fields (decode ~allow_ignored_subdirs:true)) diff --git a/src/dune/sub_dirs.mli b/src/dune/sub_dirs.mli index 6d44f8bd76b..eb10010b98b 100644 --- a/src/dune/sub_dirs.mli +++ b/src/dune/sub_dirs.mli @@ -23,6 +23,8 @@ module Status : sig ; normal : 'a } + val merge : 'a t -> 'b t -> f:('a -> 'b -> 'c) -> 'c t + val find : 'a t -> status -> 'a val to_dyn : ('a -> Dyn.t) -> 'a t -> Dyn.t @@ -38,6 +40,10 @@ module Status : sig end end +type subdir_stanzas = (Loc.t * Predicate_lang.Glob.t) option Status.Map.t + +val or_default : subdir_stanzas -> Predicate_lang.Glob.t Status.Map.t + val default : Predicate_lang.Glob.t Status.Map.t type status_map @@ -46,6 +52,21 @@ val eval : Predicate_lang.Glob.t Status.Map.t -> dirs:string list -> status_map val status : status_map -> dir:string -> Status.Or_ignored.t -val decode : - (Predicate_lang.Glob.t Status.Map.t * Dune_lang.Ast.t list) - Dune_lang.Decoder.t +module Dir_map : sig + type t + + type per_dir = + { sexps : Dune_lang.Ast.t list + ; subdir_status : subdir_stanzas + } + + val descend : t -> string -> t option + + val sub_dirs : t -> string list + + val merge : t -> t -> t + + val root : t -> per_dir +end + +val decode : Dir_map.t Dune_lang.Decoder.t diff --git a/src/dune/utop.ml b/src/dune/utop.ml index 5a9f7d8f6ee..3497764ca65 100644 --- a/src/dune/utop.ml +++ b/src/dune/utop.ml @@ -25,8 +25,8 @@ let libs_and_ppx_under_dir sctx ~db ~dir = (let open Option.O in let* dir = Path.drop_build_context dir in let+ dir = File_tree.find_dir dir in - File_tree.Dir.fold dir ~traverse:Sub_dirs.Status.Set.all ~init:([], []) - ~f:(fun dir (acc, pps) -> + File_tree.Dir.fold_dune_files dir ~init:([], []) + ~f:(fun ~basename:_ dir _dune_file (acc, pps) -> let dir = Path.Build.append_source (Super_context.build_dir sctx) diff --git a/src/stdune/path.ml b/src/stdune/path.ml index 33865aaa351..34438b6f9c2 100644 --- a/src/stdune/path.ml +++ b/src/stdune/path.ml @@ -19,6 +19,12 @@ module Fpath = struct ) end +let basename_opt ~is_root ~basename t = + if is_root t then + None + else + Some (basename t) + let is_dir_sep = if Sys.win32 || Sys.cygwin then fun c -> @@ -126,6 +132,8 @@ end = struct let is_root = equal root + let basename_opt = basename_opt ~is_root ~basename + let parent t = if is_root t then None @@ -555,6 +563,8 @@ end = struct include Fix_root (struct type nonrec w = w end) + + let basename_opt = basename_opt ~is_root ~basename end module Relative_to_source_root = struct @@ -928,6 +938,8 @@ let basename t = | In_source_dir t -> Local.basename t | External t -> External.basename t +let basename_opt = basename_opt ~is_root ~basename + let parent = function | External s -> Option.map (External.parent s) ~f:external_ | In_source_tree l -> Local.parent l |> Option.map ~f:in_source_tree diff --git a/src/stdune/path.mli b/src/stdune/path.mli index a014a520c0f..91787734c21 100644 --- a/src/stdune/path.mli +++ b/src/stdune/path.mli @@ -96,6 +96,8 @@ module Source : sig though having such paths is almost always an error. *) val is_in_build_dir : t -> bool + val descendant : t -> of_:t -> t option + val to_local : t -> Local.t end diff --git a/src/stdune/path_intf.ml b/src/stdune/path_intf.ml index bfb360ac688..a5318d53c28 100644 --- a/src/stdune/path_intf.ml +++ b/src/stdune/path_intf.ml @@ -25,6 +25,8 @@ module type S = sig val basename : t -> string + val basename_opt : t -> string option + val extend_basename : t -> suffix:string -> t module Set : sig diff --git a/test/blackbox-tests/dune.inc b/test/blackbox-tests/dune.inc index e2e6b9f7c5d..5a4989ad6a8 100644 --- a/test/blackbox-tests/dune.inc +++ b/test/blackbox-tests/dune.inc @@ -1860,6 +1860,14 @@ test-cases/strict-package-deps (progn (run dune-cram run run.t) (diff? run.t run.t.corrected))))) +(rule + (alias subdir-stanza) + (deps (package dune) (source_tree test-cases/subdir-stanza)) + (action + (chdir + test-cases/subdir-stanza + (progn (run dune-cram run run.t) (diff? run.t run.t.corrected))))) + (rule (alias subst) (deps (package dune) (source_tree test-cases/subst)) @@ -2650,6 +2658,7 @@ (alias stale-artifact-removal) (alias stdlib-compilation) (alias strict-package-deps) + (alias subdir-stanza) (alias subst) (alias syntax-versioning) (alias target-dir-alias) @@ -2911,6 +2920,7 @@ (alias stale-artifact-removal) (alias stdlib-compilation) (alias strict-package-deps) + (alias subdir-stanza) (alias subst) (alias syntax-versioning) (alias target-dir-alias) diff --git a/test/blackbox-tests/test-cases/subdir-stanza/run.t b/test/blackbox-tests/test-cases/subdir-stanza/run.t new file mode 100644 index 00000000000..949c70a4fba --- /dev/null +++ b/test/blackbox-tests/test-cases/subdir-stanza/run.t @@ -0,0 +1,73 @@ +(subdir ..) allows us to interpret stanzas in a sub directory + + $ echo "(lang dune 2.5)" > dune-project + $ cat >dune < (rule (with-stdout-to foo.txt (echo "bar"))) + > (subdir bar + > (rule (with-stdout-to foo.txt (echo "bar")))) + $ dune build ./foo.txt ./bar/foo.txt + $ cat _build/default/foo.txt + bar + + +We can use paths such as foo/bar in (subdir ..) + + $ cat >dune < (rule (with-stdout-to foo.txt (echo bar))) + > (subdir bar/baz + > (subdir final + > (rule (with-stdout-to txt (echo final)))) + > (rule (with-stdout-to foo.txt (echo bar)))) + > EOF + $ dune build bar/baz/foo.txt bar/baz/final/txt + $ cat _build/default/bar/baz/foo.txt + bar + $ cat _build/default/bar/baz/final/txt + final + +This is an error because we cannot specify data_only_dirs more than once per +dir. + + $ cat >dune < (subdir bar (data_only_dirs foo)) + > EOF + $ mkdir bar + $ echo "(data_only_dirs foo)" > bar/dune + $ dune build @all + File "bar/dune", line 1, characters 16-19: + 1 | (data_only_dirs foo) + ^^^ + Error: This stanza stanza was already specified at: + dune:1 + [1] + +Overriding dune files in the sub directory is possible: + + $ mkdir override; cd override + $ echo "(lang dune 2.5)" > dune-project + $ cat >dune < (data_only_dirs shadow) + > (subdir shadow (rule (with-stdout-to bar (echo shadow)))) + > EOF + $ mkdir shadow + $ echo "does not work" > shadow/dune + $ dune build ./shadow/bar + $ cat _build/default/shadow/bar + shadow + $ cd .. + +In conjunction with dune generated files: + + $ mkdir dune-syntax; cd dune-syntax + $ echo "(lang dune 2.5)" > dune-project + $ cat >dune < (subdir sub (rule (with-stdout-to fromparent (echo parent)))) + > EOF + $ mkdir sub + $ cat >sub/dune < (* -*- tuareg -*- *) + > let () = Jbuild_plugin.V1.send {|(rule (with-stdout-to bar (echo %{read:fromparent})))|}; + > EOF + $ dune build ./sub/bar + $ cat _build/default/sub/bar + parent