From ccf93980ba3220d09e8608a833f6921e994eff1b Mon Sep 17 00:00:00 2001 From: vemv Date: Fri, 9 Jul 2021 01:38:22 +0200 Subject: [PATCH 1/2] Parallelize `referred-syms-by-file&fullname` Part of https://github.com/clojure-emacs/refactor-nrepl/issues/230 --- CHANGELOG.md | 2 ++ src/refactor_nrepl/core.clj | 23 +++++++++++++++-------- src/refactor_nrepl/ns/libspecs.clj | 4 ++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc4e9af5..8fc2dd8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## Unreleased #### Changes +* [(Part of #230)](https://github.com/clojure-emacs/refactor-nrepl/issues/230) Parallelize various fuctionality + * This will have a noticeable improvement in e.g. clj-refactor.el's `cljr-slash` performance. * [#291](https://github.com/clojure-emacs/refactor-nrepl/issues/291): The `:ignore-errors` option will be honored in more places, making refactor-nrepl more robust in face of files not particularly meant to be part of the AST corpus. * Examples: WIP files, Moustache template files, scripts. * Upgrade Orchard diff --git a/src/refactor_nrepl/core.clj b/src/refactor_nrepl/core.clj index 7976f878..5e899824 100644 --- a/src/refactor_nrepl/core.clj +++ b/src/refactor_nrepl/core.clj @@ -99,15 +99,19 @@ (defn find-in-dir "Searches recursively under dir for files matching (pred ^File file). - Note that files which are non-existant, hidden or build-artifacts + Note that files which are non-existent, hidden or build-artifacts are pruned by this function." [pred dir] - (->> dir - file-seq - (filter (every-pred fs/exists? - (complement fs/hidden?) - pred - (complement build-artifact?))))) + (->> dir + file-seq + (pmap (fn [f] + (when ((every-pred fs/exists? + (complement fs/hidden?) + pred + (complement build-artifact?)) + f) + f))) + (filter identity))) (defn read-ns-form ([path] @@ -184,7 +188,10 @@ (defn find-in-project "Return the files in the project satisfying (pred ^File file)." [pred] - (-> find-in-dir (partial pred) (mapcat (dirs-on-classpath)) distinct)) + (->> (dirs-on-classpath) + (pmap (partial find-in-dir pred)) + (apply concat) + distinct)) (defn throw-unless-clj-file [file-path] (when-not (re-matches #".+\.clj$" file-path) diff --git a/src/refactor_nrepl/ns/libspecs.clj b/src/refactor_nrepl/ns/libspecs.clj index 68568ce4..88f3121f 100644 --- a/src/refactor_nrepl/ns/libspecs.clj +++ b/src/refactor_nrepl/ns/libspecs.clj @@ -88,10 +88,10 @@ {:clj (->> (core/find-in-project (util/with-suppressed-errors (some-fn core/clj-file? core/cljc-file?) ignore-errors?)) - (map (juxt identity (partial get-libspec-from-file-with-caching :clj))) + (pmap (juxt identity (partial get-libspec-from-file-with-caching :clj))) sym-by-file&fullname) :cljs (->> (core/find-in-project (util/with-suppressed-errors (some-fn core/cljs-file? core/cljc-file?) ignore-errors?)) - (map (juxt identity (partial get-libspec-from-file-with-caching :cljs))) + (pmap (juxt identity (partial get-libspec-from-file-with-caching :cljs))) sym-by-file&fullname)})) From 02bc6d36e192f27cc7a85e85bf6711479e32a65f Mon Sep 17 00:00:00 2001 From: vemv Date: Fri, 9 Jul 2021 16:14:48 +0200 Subject: [PATCH 2/2] PR feedback --- CHANGELOG.md | 4 +++- src/refactor_nrepl/core.clj | 1 + src/refactor_nrepl/ns/libspecs.clj | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fc2dd8b..b2843d15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ ## Unreleased #### Changes -* [(Part of #230)](https://github.com/clojure-emacs/refactor-nrepl/issues/230) Parallelize various fuctionality + +* [(Part of #230)](https://github.com/clojure-emacs/refactor-nrepl/issues/230): Parallelize various functionality * This will have a noticeable improvement in e.g. clj-refactor.el's `cljr-slash` performance. * [#291](https://github.com/clojure-emacs/refactor-nrepl/issues/291): The `:ignore-errors` option will be honored in more places, making refactor-nrepl more robust in face of files not particularly meant to be part of the AST corpus. * Examples: WIP files, Moustache template files, scripts. @@ -17,6 +18,7 @@ * Honor internal `future-cancel` calls, improving overall responsiveness and stability. ### Bugs fixed + * [#289](https://github.com/clojure-emacs/refactor-nrepl/issues/289): Fix an edge-case with involving keywords that caused find-symbol to crash. * [#305](https://github.com/clojure-emacs/refactor-nrepl/issues/305): Don't put `:as` or `:refer` on their own lines in the ns form, when the libspec is so long it causes the line to wrap. * [clojure-emacs/clj-refactor.el#459](https://github.com/clojure-emacs/clj-refactor.el/issues/459): `clean-ns` should conform to the style guide: `(:require` in the ns form should be followed by a newline. diff --git a/src/refactor_nrepl/core.clj b/src/refactor_nrepl/core.clj index 5e899824..435a9ff2 100644 --- a/src/refactor_nrepl/core.clj +++ b/src/refactor_nrepl/core.clj @@ -104,6 +104,7 @@ [pred dir] (->> dir file-seq + ;; `pmap` performs better in large projects. (pmap (fn [f] (when ((every-pred fs/exists? (complement fs/hidden?) diff --git a/src/refactor_nrepl/ns/libspecs.clj b/src/refactor_nrepl/ns/libspecs.clj index 88f3121f..496a59be 100644 --- a/src/refactor_nrepl/ns/libspecs.clj +++ b/src/refactor_nrepl/ns/libspecs.clj @@ -77,7 +77,7 @@ (defn referred-syms-by-file&fullname "Return a map of filename to a map of sym fullname to sym - the sym itself + the sym itself. Example: {:clj {\"/home/someuser/projects/some.clj\" [\"example.com/foobar\" foobar]} @@ -85,6 +85,7 @@ ([] (referred-syms-by-file&fullname false)) ([ignore-errors?] + ;; `pmap` is used as it has proved to be more efficient, both for cached and non-cached cases. {:clj (->> (core/find-in-project (util/with-suppressed-errors (some-fn core/clj-file? core/cljc-file?) ignore-errors?))