From 7f41595c60273d86d80e9da0dae7089dc8c11585 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Thu, 31 Oct 2019 21:27:25 -0600 Subject: [PATCH 01/31] Bump version number to 2.2.0 --- docs/conf.py | 4 ++-- project.clj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index aa94908..71a8fa7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.1' +version = '2.2' # The full version, including alpha/beta/rc tags. -release = '2.1.0' +release = '2.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/project.clj b/project.clj index 7952fcf..bc4e9c9 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject degasolv/degasolv "2.1.0" +(defproject degasolv/degasolv "2.2.0-SNAPSHOT" :description "Dependency tracker with an eye toward building and shipping software." :url "http://github.com/djhaskin987/degasolv" :license {:name "Eclipse Public License" From fc89340e3ff1b4058664abcd05b44aa7a9ff0c80 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Thu, 31 Oct 2019 22:03:25 -0600 Subject: [PATCH 02/31] Update changelog --- docs/changelog.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 923f1da..c08e19d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,12 +11,25 @@ and this project adheres to `Semantic Versioning`_. .. _Semantic Versioning: http://semver.org/spec/v2.0.0.html .. _Keep a Changelog: http://keepachangelog.com/en/1.0.0/ + `Unreleased`_ ------------- Added +++++ +Changed ++++++++ + +Fixed ++++++ + +`2.1.0`_ +------------- + +Added ++++++ + - Added "version suggestion", a performance enhancement allowing for minimum version selection @@ -350,7 +363,8 @@ Added - This isn't the first release, but for the purposes of these docs, it is :D -.. _Unreleased: https://github.com/djhaskin987/degasolv/compare/2.0.0...HEAD +.. _Unreleased: https://github.com/djhaskin987/degasolv/compare/2.1.0...HEAD +.. _2.1.0: https://github.com/djhaskin987/degasolv/compare/2.0.0...2.1.0 .. _2.0.0: https://github.com/djhaskin987/degasolv/compare/1.12.1...2.0.0 .. _1.12.1: https://github.com/djhaskin987/degasolv/compare/1.12.0...1.12.1 .. _1.12.0: https://github.com/djhaskin987/degasolv/compare/1.11.0...1.12.0 From 8a822d202c277e9ca744d2945609e6bb9ad22676 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sat, 2 Nov 2019 11:24:27 -0600 Subject: [PATCH 03/31] Tests pass on underlying function --- docs/roadmap.rst | 49 +++++++++++++++++++------------- src/degasolv/cli.clj | 34 ++++++++++++++++++++++ test/degasolv/cli_funcs_test.clj | 42 +++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 19 deletions(-) create mode 100644 test/degasolv/cli_funcs_test.clj diff --git a/docs/roadmap.rst b/docs/roadmap.rst index 62a1746..7440700 100644 --- a/docs/roadmap.rst +++ b/docs/roadmap.rst @@ -14,19 +14,6 @@ Future Releases - [ ] Shortened versions of all subcommands, including documentation updates. -- [ ] Environment-variable specification of options: the ability to specify - options using environment variables. Options taking a list will accept - comma-separated lists in the environment variable. Commas in strings - as values in these lists will simply not be supported in the initial version. - -- [ ] Authenticated HTTP and HTTPS connections: we will provide a way by which - HTTP and HTTPS connections are authenticated, I think using a .netrc-like - mapping between hosts and username/password pairs. Documentation should be - written around specifying username and password securely according to needs, - whether by standard input using the conventional config file mechanisms, by - environment variable, or by command line. Each has security implications that - the user needs to be aware of. - - [ ] Documentation and/or code on the topic of supporting the use case of different architectures of the same package using prioritized indexes of packages named the same with different contents. @@ -36,7 +23,7 @@ Future Releases coincide with upgrading to Clojure 1.11 because native-image doesn't work with Clojure 1.10.1 . -2.2.0 +2.3.0 ----- Package Installation Manager @@ -81,11 +68,7 @@ This database will allow us to track and manage: present any functionality would be turned off, including ignoring the installation for configuration items. An optional username and password for the connection, ``--installation-username``, and ``--installation-password``, - would be allowed. Last wins. Documentation should be written around - specifying username and password securely according to needs, whether by - standard input using the conventional config file mechanisms, by - environment variable, or by command line. Each has security implications - that the user needs to be aware of. + would be allowed. Last wins. - [ ] If the ``--installation`` option is present, the ``resolve-locations`` subcommand would take the installation into account in its resolution as @@ -175,6 +158,34 @@ implemented upon package installation, removal, creation, and verification. package, currently being installed or previously installed, installs the same resources. + +2.2.0 +----- + +- [ ] Environment-variable specification of options: the ability to specify + options using environment variables. A string will be returned as the value + of the option, unless a caret (``^``) character is present in the string, in + which case the string will be split into a list of strings using the caret + characters as boundaries. Arguments that take a boolean option will atttempt + to parse the string, expecting either the values ``true`` or ``false`` as the + value of the string. It is an error for boolean arguments (e.g., + ``DEGASOLV_ALTERNATIVES`` or ``DEGASOLV_ERROR_FORMAT``) to have any other + string. + +- [ ] Authenticated HTTP and HTTPS connections: we will provide a way by which + HTTP and HTTPS connections are authenticated, I think using a .netrc-like + mapping between hosts and username/password pairs. Documentation should be + written around specifying username and password securely according to needs, + whether by standard input using the conventional config file mechanisms, by + environment variable, or by command line. Each has security implications that + the user needs to be aware of. + +- [ ] Documentation should be written around specifying username and password + securely according to needs, whether by standard input using the conventional + config file mechanisms, by environment variable, or by command line. Each has + security implications that the user needs to be aware of. + + 2.1.0 ----- diff --git a/src/degasolv/cli.clj b/src/degasolv/cli.clj index 4d5195b..aad2763 100644 --- a/src/degasolv/cli.clj +++ b/src/degasolv/cli.clj @@ -813,6 +813,38 @@ it) (reduce merge it))) +(defn get-env-vars [env-vars] + (->> env-vars + (filter (fn [[k v]] + (re-matches #"^DEGASOLV_[A-Z_]+$" k))) + (map (fn [[k v]] + (let [option-key (as-> k it + (subs it 9) + (string/lower-case it) + (string/replace it #"_" "-") + (keyword it))] + [option-key + (if (not + (or (= option-key :alternatives) + (= option-key :error-format))) + (if (not (re-matches #".*\^.*" v)) + v + (string/split v #"\^"))) + (cond (= v "true") + true + (= v "false") + false + :else + (throw + (ex-info + (str + "Boolean options require `true` or `false` to be" + "set in environment variables") + {:option + option-key + :value-given v})))]))) + (into {}))) + (defn -main [& args] (let [{:keys [options arguments]} (parseplz! "degasolv" args cli-spec)] @@ -847,6 +879,8 @@ (:config-files global-options)) config (get-config config-files) + env-vars + (get-env-vars (System/getenv)) cli-option-packs (:option-packs global-options) selected-option-packs (if (empty? cli-option-packs) diff --git a/test/degasolv/cli_funcs_test.clj b/test/degasolv/cli_funcs_test.clj new file mode 100644 index 0000000..a71ca6b --- /dev/null +++ b/test/degasolv/cli_funcs_test.clj @@ -0,0 +1,42 @@ +(ns degasolv.cli-funcs-test + (:require [clojure.test :refer :all] + [degasolv.cli :refer :all])) + +(deftest ^:unit-tests env-vars-test + (testing "Works okay with no vars" + (is (empty? (get-env-vars nil)))) + (testing "Returns no mismatching variables" + (is (empty? (get-env-vars {"DEGASOLV_" "Didn't have anything else after the underscore" + "degasolv_a" "lowercase, doesn't count" + "DEGASOLv_b" "mixed case prefix, doesn't count" + "DEGASOLV_b" "mixed case suffix, doesn't count" + "ALTERNATIVES" "lacks prefix"})))) + (testing "Boolean arguments raise exception" + (is (thrown? + Exception + (get-env-vars {"DEGASOLV_ALTERNATIVES" "TRUE"}))) + (is (thrown? + Exception + (get-env-vars {"DEGASOLV_ERROR_FORMAT" "fAlSe"}))) + (is (thrown? + Exception + (get-env-vars {"DEGASOLV_ERROR_FORMAT" ""}))) + (is (thrown? + Exception + (get-env-vars {"DEGASOLV_ALTERNATIVES" + "literally any other value"})))) + (testing "Filters out the right ones" + (is (= + {:a "it takes all kinds" + :alternatives true + :error-format false + :conflict-strat "inclusive" + :requirements + ["a>=1,!=2;<=0" + "b|c"]} + (get-env-vars + {"DEGASOLV_A" "it takes all kinds" + "DEGASOLV_ALTERNATIVES" "true" + "DEGASOLV_ERROR_FORMAT" "false" + "DEGASOLV_CONFLICT_STRAT" "inclusive" + "DEGASOLV_REQUIREMENTS" "a>=1,!=2;<=0^b|c" }))))) From 980b4e5ceeabceb5da11927ebb693913d492cdb8 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sat, 2 Nov 2019 20:26:31 -0600 Subject: [PATCH 04/31] Add environment variables into the CLI --- src/degasolv/cli.clj | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/degasolv/cli.clj b/src/degasolv/cli.clj index aad2763..ad2315e 100644 --- a/src/degasolv/cli.clj +++ b/src/degasolv/cli.clj @@ -890,8 +890,11 @@ (as-> selected-option-packs it (mapv available-option-packs it) - (into [subcommand-option-defaults] it) - (conj it (dissoc config :option-packs)) + [ + subcommand-option-defaults + it + (dissoc config :option-packs) + env-vars] (conj it (into {} (filter #(not (nil? (second %))) From 6c26a3069faf352d46864dd47054d8b85cd7935a Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Tue, 5 Nov 2019 22:06:36 -0700 Subject: [PATCH 05/31] Clip in the wall: code complete for get-env-vars --- src/degasolv/cli.clj | 253 ++++++++++++++++----------- test/resources/scripts/test-env-vars | 104 +++++++++++ 2 files changed, 259 insertions(+), 98 deletions(-) create mode 100644 test/resources/scripts/test-env-vars diff --git a/src/degasolv/cli.clj b/src/degasolv/cli.clj index ad2315e..060d490 100644 --- a/src/degasolv/cli.clj +++ b/src/degasolv/cli.clj @@ -38,6 +38,7 @@ (:version %1) (:version %2)))))) + (def ^:private version-comparators @@ -814,108 +815,164 @@ (reduce merge it))) (defn get-env-vars [env-vars] - (->> env-vars - (filter (fn [[k v]] - (re-matches #"^DEGASOLV_[A-Z_]+$" k))) - (map (fn [[k v]] - (let [option-key (as-> k it - (subs it 9) - (string/lower-case it) - (string/replace it #"_" "-") - (keyword it))] - [option-key - (if (not - (or (= option-key :alternatives) - (= option-key :error-format))) - (if (not (re-matches #".*\^.*" v)) - v - (string/split v #"\^"))) - (cond (= v "true") - true - (= v "false") - false - :else - (throw - (ex-info - (str - "Boolean options require `true` or `false` to be" - "set in environment variables") - {:option - option-key - :value-given v})))]))) - (into {}))) + (letfn [(map-transform [v] + (->> (string/split v #"\^") + (map #(into [] (string/split % #":"))) + (into {}))) + (config-transform [f] + (->> f + (list-transform) + (map (fn [x] + {:file (io/file x) + :read-fn tag/read-string})))) + (json-config-transform [f] + (->> f + (list-transform) + (map (fn [x] + {:file (io/file x) + :read-fn #(json/parse-string % true)})))) + (list-transform [v] + (string/split v #"\^")) + (boolean-transform [v] + (cond (= v "true") + true + (= v "false") + false + :else + (throw + (ex-info + (str + "Boolean options require `true` or `false` to be" + "set in environment variables") + {:value-given v}))))] + (let [transform-functions + { + :alternatives boolean-transform + :config-files config-transform + :error-format boolean-transform + :json-config-files json-config-transform + :meta map-transform + :option-packs list-transform + :present-packages list-transform + :repositories list-transform + :requirements list-transform + }] + (->> env-vars + (filter (fn [[k v]] + (re-matches #"^DEGASOLV_[A-Z_]+$" k))) + (map (fn [[k v]] + (let [option-key (as-> k it + (subs it 9) + (string/lower-case it) + (string/replace it #"_" "-") + (keyword it))] + (if-let [tf (transform-functions option-key)] + [option-key (tf v)] + [option-key v]))) + (into {})))))) (defn -main [& args] - (let [{:keys [options arguments]} - (parseplz! "degasolv" args cli-spec)] - (let [global-options options - subcommand (first arguments) - subcommand-cli (:subcommands cli-spec) - subcmd-cli (if (not (= subcommand "display-config")) - (get subcommand-cli subcommand) - ; this grabs all other options as part of display-config - (as-> subcommand-cli it - (vals it) - (map :cli it) - (filter #(not (nil? %)) it) - (apply concat it) - (map #(concat [nil] (subvec % 1)) it) - (map #(do [(second %) %]) it) - (into {} it) - (vals it) - (assoc - (get subcommand-cli subcommand) - :cli - it) - it))] - (when (nil? subcmd-cli) - (exit 1 (error-msg [(str "Unknown command: " subcommand)]))) - (let [{:keys [options arguments]} - (parseplz! subcommand (rest arguments) subcmd-cli)] - (let [config-files - (if (empty? (:config-files global-options)) - [{:file (io/file "./degasolv.edn") - :read-fn tag/read-string}] - (:config-files global-options)) - config - (get-config config-files) - env-vars - (get-env-vars (System/getenv)) - cli-option-packs (:option-packs global-options) - selected-option-packs - (if (empty? cli-option-packs) - (into [] (:option-packs config)) - cli-option-packs) - effective-options - (as-> - selected-option-packs it - (mapv available-option-packs it) - [ - subcommand-option-defaults + (let [env-vars + (get-env-vars (System/getenv)) + {:keys [options arguments]} + (parseplz! "degasolv" args cli-spec) + global-options (reduce merge [env-vars options]) + subcommand (first arguments) + subcommand-cli (:subcommands cli-spec) + subcmd-cli (if (not (= subcommand "display-config")) + (get subcommand-cli subcommand) + ; this grabs all other options as part of display-config + (as-> subcommand-cli it + (vals it) + (map :cli it) + (filter #(not (nil? %)) it) + (apply concat it) + (map #(concat [nil] (subvec % 1)) it) + (map #(do [(second %) %]) it) + (into {} it) + (vals it) + (assoc + (get subcommand-cli subcommand) + :cli + it) + it))] + (when (nil? subcmd-cli) + (exit 1 (error-msg [(str "Unknown command: " subcommand)]))) + (let [{:keys [options arguments]} + (parseplz! subcommand (rest arguments) subcmd-cli) + config-files + (reduce (fn [[seen result] ^java.io.File new] + (let [abs-path (.getAbsolutePath new)] + (if (or (seen abs-path) + (not (.exists new))) + [seen result] + [(conj seen abs-path) + (conj result new)]))) + [#{} []] + (into [] + (if-let [app-data (System/getenv "AppData")] + [{:file (io/file (string/join java.io.File/pathSeparator + [app-data + "degasolv" + "config.json"])) + :read-fn #(json/parse-string % true)} + {:file (io/file (string/join java.io.File/pathSeparator + [app-data + "degasolv" + "config.edn"])) + :read-fn tag/read-string}]) + (if-let [home (System/getenv "HOME")] + [{:file (io/file (string/join java.io.File/pathSeparator + [home + ".degasolv.json"])) + :read-fn #(json/parse-string % true)} + {:file (io/file (string/join java.io.File/pathSeparator + [home + ".degasolv.edn"])) + :read-fn tag/read-string}]) + [{:file (io/file (string/join java.io.File/pathSeparator + ["." "degasolv.json"])) + :read-fn #(json/parse-string % true)} + {:file (io/file (string/join java.io.File/pathSeparator + ["." "degasolv.edn"])) + :read-fn tag/read-string}] + (:config-files global-options))) + config + (get-config config-files) + cli-option-packs (:option-packs global-options) + selected-option-packs + (if (empty? cli-option-packs) + (into [] (:option-packs config)) + cli-option-packs) + effective-options + (as-> + selected-option-packs it + (mapv available-option-packs it) + [subcommand-option-defaults + it + (dissoc config :option-packs) + env-vars] + (conj it (into {} + (filter + #(not (nil? (second %))) + (seq options)))) + (reduce merge (hash-map) it) + (if (not (:version-comparison it)) + (assoc it - (dissoc config :option-packs) - env-vars] - (conj it (into {} - (filter - #(not (nil? (second %))) - (seq options)))) - (reduce merge (hash-map) it) - (if (not (:version-comparison it)) - (assoc - it - :version-comparison - (if (get-in package-systems - [(:package-system it) :version-comparison]) - (get-in package-systems + :version-comparison + (if (get-in package-systems [(:package-system it) :version-comparison]) - (get-in package-systems - [(:package-system subcommand-option-defaults) - :version-comparison]))) - it))] - (check-required! effective-options subcmd-cli) - ((:function subcmd-cli) - effective-options - arguments))))) + (get-in package-systems + [(:package-system it) :version-comparison]) + (get-in package-systems + [(:package-system subcommand-option-defaults) + :version-comparison]))) + it))] + (check-required! effective-options subcmd-cli) + ((:function subcmd-cli) + effective-options + arguments))) ;; Subproc package system forks processess, ;; which causes the VM to hang unless this is called ;; https://dev.clojure.org/jira/browse/CLJ-959 diff --git a/test/resources/scripts/test-env-vars b/test/resources/scripts/test-env-vars new file mode 100644 index 0000000..f82e6d1 --- /dev/null +++ b/test/resources/scripts/test-env-vars @@ -0,0 +1,104 @@ +#!/bin/sh +set -eoux + +export POSIXLY_CORRECT=1 + +if [ ! -f './project.clj' -a ! -f './build.boot' ] +then + echo "This script must be run from the root of the project." +fi + +root_path=${PWD} +test_home=test/resources/data/test-env-vars + +name=$(lein print :name | sed 's|"||g') +version=$(lein print :version | sed 's|"||g') + +rm -rf "${test_home}" +mkdir -p "${test_home}" +cd "${test_home}" + +# First, generate the needed cards for resolution. + +DEGASOLV_ID="a" DEGASOLV_REQUIREMENTS="^b" + +java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ + generate-card \ + -i "a" \ + -v "1.0.0" \ + -r "b" \ + -l "https://example.com/repo/a-1.0.0.zip" \ + -C $PWD/a-1.0.0.zip.dscard + +java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ + generate-card \ + -i "b" \ + -v "2.3.0" \ + -l "https://example.com/repo/b-2.3.0.zip" \ + -C $PWD/b-2.3.0.zip.dscard + +java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar generate-repo-index -d $PWD -I $PWD/index.dsrepo + +java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ + resolve-locations \ + -R ${PWD}/index.dsrepo \ + --output-format json \ + --requirement "a<=1.0.0" + +if java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ + resolve-locations \ + -R ${PWD}/index.dsrepo \ + --output-format json \ + --requirement "z<=1.0.0" +then + exit 1 +fi + +if ! java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ + resolve-locations \ + -R ${PWD}/index.dsrepo \ + --enable-error-format \ + --output-format json \ + --requirement "z<=1.0.0" | jq .problems +then + exit 1 +fi + + +java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ + resolve-locations \ + --output-format edn \ + -R ${PWD}/index.dsrepo \ + --requirement "a<=1.0.0" + +if java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ + resolve-locations \ + --output-format edn \ + -R ${PWD}/index.dsrepo \ + --requirement "z<=1.0.0" +then + exit 1 +fi + +if ! java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ + resolve-locations \ + --enable-error-format \ + --output-format edn \ + -R ${PWD}/index.dsrepo \ + --requirement "z<=1.0.0" | grep -q -E ':problems' +then + exit 1 +fi + +java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ + resolve-locations \ + -R ${PWD}/index.dsrepo \ + --requirement "a<=1.0.0" + +if java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ + resolve-locations \ + -R ${PWD}/index.dsrepo \ + --requirement "z<=1.0.0" +then + exit 1 +fi From 53a6ad1410eb9cfc98773c2b16ec7483a16d705a Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Wed, 6 Nov 2019 18:09:00 -0700 Subject: [PATCH 06/31] get-env-vars passes unit tests --- src/degasolv/cli.clj | 27 ++++++++++++++------ test/degasolv/cli_funcs_test.clj | 42 +++++++++++++++++++++----------- 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/degasolv/cli.clj b/src/degasolv/cli.clj index 060d490..aac6e86 100644 --- a/src/degasolv/cli.clj +++ b/src/degasolv/cli.clj @@ -814,10 +814,12 @@ it) (reduce merge it))) -(defn get-env-vars [env-vars] +(defn + get-env-vars + [env-vars] (letfn [(map-transform [v] (->> (string/split v #"\^") - (map #(into [] (string/split % #":"))) + (map #(into [] (string/split % #"="))) (into {}))) (config-transform [f] (->> f @@ -856,10 +858,11 @@ :present-packages list-transform :repositories list-transform :requirements list-transform - }] - (->> env-vars + } + ] + (as-> env-vars it (filter (fn [[k v]] - (re-matches #"^DEGASOLV_[A-Z_]+$" k))) + (re-matches #"^DEGASOLV_[A-Z_]+$" k)) it) (map (fn [[k v]] (let [option-key (as-> k it (subs it 9) @@ -868,8 +871,18 @@ (keyword it))] (if-let [tf (transform-functions option-key)] [option-key (tf v)] - [option-key v]))) - (into {})))))) + [option-key v]))) it) + (into {} it) + (if (or (:config-files it) (:json-config-files it)) + (assoc + (dissoc it :json-config-files) + :config-files + (reduce + into + [] + [(:config-files it) + (:json-config-files it)])) + it))))) (defn -main [& args] (let [env-vars diff --git a/test/degasolv/cli_funcs_test.clj b/test/degasolv/cli_funcs_test.clj index a71ca6b..510b4c4 100644 --- a/test/degasolv/cli_funcs_test.clj +++ b/test/degasolv/cli_funcs_test.clj @@ -26,17 +26,31 @@ (get-env-vars {"DEGASOLV_ALTERNATIVES" "literally any other value"})))) (testing "Filters out the right ones" - (is (= - {:a "it takes all kinds" - :alternatives true - :error-format false - :conflict-strat "inclusive" - :requirements - ["a>=1,!=2;<=0" - "b|c"]} - (get-env-vars - {"DEGASOLV_A" "it takes all kinds" - "DEGASOLV_ALTERNATIVES" "true" - "DEGASOLV_ERROR_FORMAT" "false" - "DEGASOLV_CONFLICT_STRAT" "inclusive" - "DEGASOLV_REQUIREMENTS" "a>=1,!=2;<=0^b|c" }))))) + (let [result (get-env-vars + {"DEGASOLV_A" "it takes all kinds" + "DEGASOLV_ALTERNATIVES" "true" + "DEGASOLV_ERROR_FORMAT" "false" + "DEGASOLV_CONFLICT_STRAT" "inclusive" + "DEGASOLV_REQUIREMENTS" "a>=1,!=2;<=0^b|c" + "DEGASOLV_REPOSITORIES" "https://a^https://b" + "DEGASOLV_CONFIG_FILES" "a.edn^b.edn" + "DEGASOLV_JSON_CONFIG_FILES" "a.json^b.json" + "DEGASOLV_META" "a=1^b=2^c=3" })] + (is (= + { + :a "it takes all kinds" + :alternatives true + :error-format false + :conflict-strat "inclusive" + :requirements + ["a>=1,!=2;<=0" + "b|c"] + :repositories ["https://a", "https://b"] + :meta { + "a" "1" + "b" "2" + "c" "3" + } + } + (dissoc result :config-files))) + (is (= (count (:config-files result)) 4))))) From 7905294d0516b6e223f7d6c60da7b3afc72d33c8 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Thu, 7 Nov 2019 08:13:53 -0700 Subject: [PATCH 07/31] display-config works now --- docs/changelog.rst | 9 +++ src/degasolv/cli.clj | 109 ++++++++++++++++----------- test/degasolv/cli_funcs_test.clj | 33 ++++++++ test/resources/scripts/test-env-vars | 99 ++++++------------------ 4 files changed, 131 insertions(+), 119 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c08e19d..6bd084d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,12 +18,21 @@ and this project adheres to `Semantic Versioning`_. Added +++++ +- Configuration file tower +- Environment variable support: environment variables are consulted first, + but "matter" in between config files and command-line options. + + Changed +++++++ Fixed +++++ +- option packs are expanded at the level upon which they are defined. TODO NEEDS DOCS + +- json-config-files are collapsed to config files at the end + `2.1.0`_ ------------- diff --git a/src/degasolv/cli.clj b/src/degasolv/cli.clj index aac6e86..9569ef5 100644 --- a/src/degasolv/cli.clj +++ b/src/degasolv/cli.clj @@ -872,7 +872,7 @@ (if-let [tf (transform-functions option-key)] [option-key (tf v)] [option-key v]))) it) - (into {} it) + (into {} it) (if (or (:config-files it) (:json-config-files it)) (assoc (dissoc it :json-config-files) @@ -884,12 +884,20 @@ (:json-config-files it)])) it))))) +(defn expand-option-packs + [options] + (as-> (:option-packs options) it + (mapv available-option-packs it) + (into {} it) + (merge it options) + (dissoc it :option-packs))) + (defn -main [& args] (let [env-vars (get-env-vars (System/getenv)) {:keys [options arguments]} (parseplz! "degasolv" args cli-spec) - global-options (reduce merge [env-vars options]) + global-options options subcommand (first arguments) subcommand-cli (:subcommands cli-spec) subcmd-cli (if (not (= subcommand "display-config")) @@ -914,57 +922,68 @@ (let [{:keys [options arguments]} (parseplz! subcommand (rest arguments) subcmd-cli) config-files - (reduce (fn [[seen result] ^java.io.File new] + (reduce (fn [[seen result] {^java.io.File new :file read-fn :read-fn + :as unit}] (let [abs-path (.getAbsolutePath new)] (if (or (seen abs-path) - (not (.exists new))) - [seen result] - [(conj seen abs-path) - (conj result new)]))) + (not (.exists new))) + [seen result] + [(conj seen abs-path) + (conj result unit)]))) [#{} []] - (into [] - (if-let [app-data (System/getenv "AppData")] - [{:file (io/file (string/join java.io.File/pathSeparator - [app-data - "degasolv" - "config.json"])) - :read-fn #(json/parse-string % true)} - {:file (io/file (string/join java.io.File/pathSeparator - [app-data - "degasolv" - "config.edn"])) - :read-fn tag/read-string}]) - (if-let [home (System/getenv "HOME")] - [{:file (io/file (string/join java.io.File/pathSeparator - [home - ".degasolv.json"])) - :read-fn #(json/parse-string % true)} - {:file (io/file (string/join java.io.File/pathSeparator - [home - ".degasolv.edn"])) - :read-fn tag/read-string}]) - [{:file (io/file (string/join java.io.File/pathSeparator - ["." "degasolv.json"])) - :read-fn #(json/parse-string % true)} - {:file (io/file (string/join java.io.File/pathSeparator - ["." "degasolv.edn"])) - :read-fn tag/read-string}] - (:config-files global-options))) + (reduce + into + [] + [(if-let [app-data (System/getenv "AppData")] + [{:file (io/file (string/join java.io.File/pathSeparator + [app-data + "degasolv" + "config.json"])) + :read-fn #(json/parse-string % true)} + {:file (io/file (string/join java.io.File/pathSeparator + [app-data + "degasolv" + "config.edn"])) + :read-fn tag/read-string}]) + (if-let [home (System/getenv "HOME")] + [{:file (io/file (string/join java.io.File/pathSeparator + [home + ".degasolv.json"])) + :read-fn #(json/parse-string % true)} + {:file (io/file (string/join java.io.File/pathSeparator + [home + ".degasolv.edn"])) + :read-fn tag/read-string}]) + [{:file (io/file (string/join java.io.File/pathSeparator + ["." "degasolv.json"])) + :read-fn #(json/parse-string % true)} + {:file (io/file (string/join java.io.File/pathSeparator + ["." "degasolv.edn"])) + :read-fn tag/read-string}] + (:config-files env-vars) + (:config-files global-options)])) config (get-config config-files) - cli-option-packs (:option-packs global-options) - selected-option-packs - (if (empty? cli-option-packs) - (into [] (:option-packs config)) - cli-option-packs) + expanded-cfg (expand-option-packs config) + expanded-env (expand-option-packs env-vars) + expanded-cli (expand-option-packs global-options) effective-options (as-> - selected-option-packs it - (mapv available-option-packs it) [subcommand-option-defaults - it - (dissoc config :option-packs) - env-vars] + expanded-cfg + (reduce dissoc + expanded-cli + [:config-files + :json-config-files + :edn-config-files + ]) + (reduce dissoc + expanded-env + [:config-files + :json-config-files + :edn-config-files + ]) + ] it (conj it (into {} (filter #(not (nil? (second %))) diff --git a/test/degasolv/cli_funcs_test.clj b/test/degasolv/cli_funcs_test.clj index 510b4c4..03bf9eb 100644 --- a/test/degasolv/cli_funcs_test.clj +++ b/test/degasolv/cli_funcs_test.clj @@ -2,6 +2,39 @@ (:require [clojure.test :refer :all] [degasolv.cli :refer :all])) +(deftest ^:unit-tests expand-option-packs-test + (testing "Empty option packs test" + (is (empty? (expand-option-packs {})))) + (testing "Disparate option packs" + (is (empty? + (expand-option-packs + {:option-packs ["a"]})))) + (testing "Normal option packs expansion" + (is (= + { + :conflict-strat "prioritized" + :resolve-strat "fast" + :alternatives true + } + (expand-option-packs + { + :option-packs ["firstfound-version-mode"] + :alternatives true + })))) + (testing "Option packs override each other" + (is (= + { + :conflict-strat "prioritized" + :resolve-strat "fast" + :alternatives false + :error-format false + :list-strat "as-set" + } + (expand-option-packs + {:option-packs ["v1" + "multi-version-mode" + "firstfound-version-mode"]}))))) + (deftest ^:unit-tests env-vars-test (testing "Works okay with no vars" (is (empty? (get-env-vars nil)))) diff --git a/test/resources/scripts/test-env-vars b/test/resources/scripts/test-env-vars index f82e6d1..ce46f35 100644 --- a/test/resources/scripts/test-env-vars +++ b/test/resources/scripts/test-env-vars @@ -20,85 +20,36 @@ cd "${test_home}" # First, generate the needed cards for resolution. -DEGASOLV_ID="a" DEGASOLV_REQUIREMENTS="^b" - -java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ - generate-card \ - -i "a" \ - -v "1.0.0" \ - -r "b" \ - -l "https://example.com/repo/a-1.0.0.zip" \ - -C $PWD/a-1.0.0.zip.dscard - +DEGASOLV_ID="a" \ + DEGASOLV_REQUIREMENTS="b" \ + DEGASOLV_VERSION="1.0.0" \ + DEGASOLV_LOCATION="https://example.com/repo/a-1.0.0.zip" \ + DEGASOLV_CARD_FILE=$PWD/a-1.0.0.zip.dscard \ + java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ + generate-card + +cat > ./config.json << JSON_CONFIG +{ + "requirements": [], + "id": "b", + "location": "https://example.com/repo/b-2.3.0.zip" +} +JSON_CONFIG +cat > ./config.edn << EDN_CONFIG +{ + :version "2.3.0" + :requirements ["c"] +} +EDN_CONFIG + DEGASOLV_CONFIG_FILES="./config.edn" \ + DEGASOLV_JSON_CONFIG_FILES="./config.json" \ + DEGASOLV_CARD_FILE=$PWD/b-2.3.0.zip.dscard \ java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ - generate-card \ - -i "b" \ - -v "2.3.0" \ - -l "https://example.com/repo/b-2.3.0.zip" \ - -C $PWD/b-2.3.0.zip.dscard + generate-card java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar generate-repo-index -d $PWD -I $PWD/index.dsrepo java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ resolve-locations \ -R ${PWD}/index.dsrepo \ - --output-format json \ - --requirement "a<=1.0.0" - -if java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ - resolve-locations \ - -R ${PWD}/index.dsrepo \ - --output-format json \ - --requirement "z<=1.0.0" -then - exit 1 -fi - -if ! java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ - resolve-locations \ - -R ${PWD}/index.dsrepo \ - --enable-error-format \ - --output-format json \ - --requirement "z<=1.0.0" | jq .problems -then - exit 1 -fi - - -java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ - resolve-locations \ - --output-format edn \ - -R ${PWD}/index.dsrepo \ --requirement "a<=1.0.0" - -if java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ - resolve-locations \ - --output-format edn \ - -R ${PWD}/index.dsrepo \ - --requirement "z<=1.0.0" -then - exit 1 -fi - -if ! java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ - resolve-locations \ - --enable-error-format \ - --output-format edn \ - -R ${PWD}/index.dsrepo \ - --requirement "z<=1.0.0" | grep -q -E ':problems' -then - exit 1 -fi - -java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ - resolve-locations \ - -R ${PWD}/index.dsrepo \ - --requirement "a<=1.0.0" - -if java -jar ${root_path}/target/uberjar/${name}-${version}-standalone.jar \ - resolve-locations \ - -R ${PWD}/index.dsrepo \ - --requirement "z<=1.0.0" -then - exit 1 -fi From 6ea7b4a890eb7ecff6a90986694d6ecabefea25a Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Thu, 7 Nov 2019 17:45:52 -0700 Subject: [PATCH 08/31] Fix config file tower behavior --- src/degasolv/cli.clj | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/degasolv/cli.clj b/src/degasolv/cli.clj index 9569ef5..9fa138d 100644 --- a/src/degasolv/cli.clj +++ b/src/degasolv/cli.clj @@ -922,7 +922,7 @@ (let [{:keys [options arguments]} (parseplz! subcommand (rest arguments) subcmd-cli) config-files - (reduce (fn [[seen result] {^java.io.File new :file read-fn :read-fn + (get (reduce (fn [[seen result] {^java.io.File new :file read-fn :read-fn :as unit}] (let [abs-path (.getAbsolutePath new)] (if (or (seen abs-path) @@ -935,33 +935,39 @@ into [] [(if-let [app-data (System/getenv "AppData")] - [{:file (io/file (string/join java.io.File/pathSeparator + [ + {:file (io/file (string/join java.io.File/separator [app-data "degasolv" - "config.json"])) - :read-fn #(json/parse-string % true)} - {:file (io/file (string/join java.io.File/pathSeparator + "config.edn"])) + :read-fn tag/read-string} + {:file (io/file (string/join java.io.File/separator [app-data "degasolv" - "config.edn"])) - :read-fn tag/read-string}]) + "config.json"])) + :read-fn #(json/parse-string % true)} + ]) (if-let [home (System/getenv "HOME")] - [{:file (io/file (string/join java.io.File/pathSeparator + [ + {:file (io/file (string/join java.io.File/separator + [home + ".degasolv.edn"])) + :read-fn tag/read-string} + {:file (io/file (string/join java.io.File/separator [home ".degasolv.json"])) :read-fn #(json/parse-string % true)} - {:file (io/file (string/join java.io.File/pathSeparator - [home - ".degasolv.edn"])) - :read-fn tag/read-string}]) - [{:file (io/file (string/join java.io.File/pathSeparator + ]) + [ + {:file (io/file (string/join java.io.File/separator + ["." "degasolv.edn"])) + :read-fn tag/read-string} + {:file (io/file (string/join java.io.File/separator ["." "degasolv.json"])) :read-fn #(json/parse-string % true)} - {:file (io/file (string/join java.io.File/pathSeparator - ["." "degasolv.edn"])) - :read-fn tag/read-string}] + ] (:config-files env-vars) - (:config-files global-options)])) + (:config-files global-options)])) 1) config (get-config config-files) expanded-cfg (expand-option-packs config) From 5c8da7f4b91d2928fd626844528b6256f9ff5d28 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Thu, 7 Nov 2019 17:48:06 -0700 Subject: [PATCH 09/31] Fix test-env-vars --- .gitignore | 1 + test/resources/scripts/test-env-vars | 0 2 files changed, 1 insertion(+) mode change 100644 => 100755 test/resources/scripts/test-env-vars diff --git a/.gitignore b/.gitignore index 9c38d11..f80083f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ /test/resources/data/grill/ /test/resources/data/meta/ /test/resources/data/test-index-sort-order/ +/test/resources/data/test-env-vars/ /test/resources/data/test-json-config/ /test/resources/data/test-list-strat/ /test/resources/data/test-option-packs/ diff --git a/test/resources/scripts/test-env-vars b/test/resources/scripts/test-env-vars old mode 100644 new mode 100755 From 5703eb2e8fef2dab052f748e448a79ce2f77ffe1 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Fri, 8 Nov 2019 08:29:01 -0700 Subject: [PATCH 10/31] Update the docs --- docs/changelog.rst | 162 +++++++------- docs/command-reference.rst | 445 +++++++++++++++++++++++++++---------- 2 files changed, 413 insertions(+), 194 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6bd084d..1e0c222 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,10 +18,15 @@ and this project adheres to `Semantic Versioning`_. Added +++++ -- Configuration file tower -- Environment variable support: environment variables are consulted first, - but "matter" in between config files and command-line options. +* Configuration file tower: Degasolv looks for config files in pre-defined + locations on the file system if they exist, see + :ref:`Default Configuration Files ` +* Environment variable support: environment variables are consulted first, + but are merged into the option map AFTER config file material and BEFORE + any given command line options, see + :ref:`Environment Variables ` and + :ref:`How Options are Gathered ` Changed +++++++ @@ -29,9 +34,12 @@ Changed Fixed +++++ -- option packs are expanded at the level upon which they are defined. TODO NEEDS DOCS - -- json-config-files are collapsed to config files at the end +* Option packs are expanded at the level upon which they are defined, then the + options are merged together. This seemed like a bug. Though this change is + breaking, we made an exception and released it as part of a minor patch + release, hoping that it would unbreak more than it would break, and knowing + if it broke anybody that they wouldn't be left without a way to fix their + scripts. See :ref:`Option Packs ` for more information. `2.1.0`_ ------------- @@ -39,16 +47,16 @@ Fixed Added +++++ -- Added "version suggestion", a performance enhancement allowing for minimum +* Added "version suggestion", a performance enhancement allowing for minimum version selection -- Added the ``:index-sort-order`` option to ``generate-repo-index``, +* Added the ``:index-sort-order`` option to ``generate-repo-index``, allowing users to specify ``ascending`` or ``descending``. Previously only ``descending`` was supported. With this new option, users will be able to use Degasolv in a minimum version selection configuration. -- If the reason for failure in the ``resolve-dependencies`` function +* If the reason for failure in the ``resolve-dependencies`` function is ``:present-package-conflict``, add a key ``:package-present-by`` with value as either ``:found`` meaning the package was in conflict with a package found during resolution, or with value as ``:given`` meaning the package @@ -57,20 +65,20 @@ Added Changed +++++++ -- Removed the deprecated functions ``->requirement``, ``->package``, and +* Removed the deprecated functions ``->requirement``, ``->package``, and ``->version-predicate`` from usage in the code base. -- Removed ``dbg2`` macro in favor of keeping ``dbg`` +* Removed ``dbg2`` macro in favor of keeping ``dbg`` Fixed +++++ -- Standardized capitalization of the word "Degasolv" to be title case for +* Standardized capitalization of the word "Degasolv" to be title case for consistency in the documentation unless it is in a code snippet. -- Fix #16 +* Fix #16 -- Fix spec for package id's. This should have the effect of enforcing that +* Fix spec for package id's. This should have the effect of enforcing that names should not have ``>``, ``<``, ``!``, ``=``, ``,``, ``;``, or ``|`` characters in them. This simply has the effect of changing the error message, as this was never allowed but handled poorly. @@ -81,33 +89,33 @@ Fixed Added +++++ -- Documentation saying what return codes are given and what they mean. +* Documentation saying what return codes are given and what they mean. -- For #15, added ability to specify output format for ``display-config``. +* For #15, added ability to specify output format for ``display-config``. -- Added :ref:`3rd Party Licenses <3rd-party-licenses>` document +* Added :ref:`3rd Party Licenses <3rd-party-licenses>` document Changed +++++++ -- In the docs, ``java -jar degasolv--standalone.jar`` changed to +* In the docs, ``java -jar degasolv--standalone.jar`` changed to ``degasolv`` with added note for clarity -- For #13, return code for ``resolve-locations`` changed to 3 when dependency +* For #13, return code for ``resolve-locations`` changed to 3 when dependency resolutions occur to distinguish them from normal "you got the argument string wrong" errors -- Default ``--{enable|disable}-error-format`` set to ``enabled`` for +* Default ``--{enable|disable}-error-format`` set to ``enabled`` for :ref:`resolve-locations ` and :ref:`query-repo `. -- Default for ``--list-strat`` option for :ref:`resolve-locations +* Default for ``--list-strat`` option for :ref:`resolve-locations ` set to ``lazy``, a much saner default. -- Option pack ``v1`` :ref:`added ` to help administrators +* Option pack ``v1`` :ref:`added ` to help administrators keep compatibility with version 1 of Degasolv if required. -- Default for the ``--version-comparison`` option when ``--package-system`` +* Default for the ``--version-comparison`` option when ``--package-system`` is ``degasolv`` set to ``semver`` for :ref:`generate-repo-index ` (option :ref:`here `), :ref:`resolve-locations @@ -115,24 +123,24 @@ Changed and :ref:`query-repo ` (option :ref:`here `). -- Removed less-than-useful warning about absent config files. +* Removed less-than-useful warning about absent config files. Fixed +++++ -- Fixed #14, "Degasolv pulls in X" +* Fixed #14, "Degasolv pulls in X" -- Fixed bug where ``display-config`` didn't allow the user +* Fixed bug where ``display-config`` didn't allow the user to specify valid options for other things, now it does -- Fixed bug where index.dsrepo didn't generate anything except +* Fixed bug where index.dsrepo didn't generate anything except an empty map inside the file. This was *completely* broken. -- Fixed #6, "If one config file fails to load, the rest do as well" +* Fixed #6, "If one config file fails to load, the rest do as well" -- Fixed #9, "Heading for 'Specifying Subproc Executable' is wrong in docs" +* Fixed #9, "Heading for 'Specifying Subproc Executable' is wrong in docs" -- Fixed #10, "How do you specify requirements of a package (deps) in the output +* Fixed #10, "How do you specify requirements of a package (deps) in the output of a subproc to Degasolv?" `1.12.1`_ @@ -144,13 +152,13 @@ artifacts were created.* Added +++++ -- Added authors file to docs +* Added authors file to docs Fixed +++++ -- Fixed changelog so that the 1.12.0 release was present in the document -- Fixed package system subheaders in command reference +* Fixed changelog so that the 1.12.0 release was present in the document +* Fixed package system subheaders in command reference `1.12.0`_ --------- @@ -158,17 +166,17 @@ Fixed Added +++++ -- Added the ``--{enable|disable}-error-format`` options to +* Added the ``--{enable|disable}-error-format`` options to :ref:`resolve-locations ` and :ref:`query-repo `. -- Added the ``--package-system subproc`` option, together with its +* Added the ``--package-system subproc`` option, together with its :ref:`subproc-exe` and :ref:`subproc-output-format` options. -- Added the ``--json-config`` :ref:`global option ` allowing users +* Added the ``--json-config`` :ref:`global option ` allowing users to use JSON config files -- Added the ``--list-strat`` option to :ref:`resolve-locations +* Added the ``--list-strat`` option to :ref:`resolve-locations `, allowing users to have their dependencies listed in a sane order. @@ -178,7 +186,7 @@ Changed Fixed +++++ -- JSON/EDN output for ``query-repo`` erroneously listed the subcommand as +* JSON/EDN output for ``query-repo`` erroneously listed the subcommand as ``resolve-locations``. Fixed. `1.11.0`_ @@ -186,20 +194,20 @@ Fixed Added +++++ -- Added the ``--meta`` :ref:`option ` to +* Added the ``--meta`` :ref:`option ` to :ref:`generate-card ` -- Added metadata a la ``--meta`` to the ``apt`` :ref:`package system +* Added metadata a la ``--meta`` to the ``apt`` :ref:`package system ` (experimental) -- Added the ``edn`` :ref:`output format option ` for +* Added the ``edn`` :ref:`output format option ` for the ``resolve-locations`` :ref:`subcommand ` -- Added the ``--output-format`` :ref:`option +* Added the ``--output-format`` :ref:`option ` to the :ref:`query-repo ` command Changed +++++++ -- Overhauled the documentation option look and feel; ensured that +* Overhauled the documentation option look and feel; ensured that options themselves had a linking target (header) `1.10.0`_ @@ -207,7 +215,7 @@ Changed Added +++++ -- Added the ``--output-format`` :ref:`option ` to +* Added the ``--output-format`` :ref:`option ` to :ref:`resolve-locations ` `1.9.0`_ @@ -215,14 +223,14 @@ Added Added +++++ -- Added the :ref:`pessimistic greater-than ` comparison operator ``><``. +* Added the :ref:`pessimistic greater-than ` comparison operator ``><``. Fixed +++++ -- Removed validation from the config file option, allowing it to be a URL or +* Removed validation from the config file option, allowing it to be a URL or anything else. -- If no arguments are given, the help screen is now printed instead of a wierd +* If no arguments are given, the help screen is now printed instead of a wierd error. `1.8.0`_ @@ -230,43 +238,43 @@ Fixed Added +++++ -- Distribution is now done via RPM and Debian package as well as JAR fil -- Added the ``--version-comparison`` option to +* Distribution is now done via RPM and Debian package as well as JAR fil +* Added the ``--version-comparison`` option to :ref:`generate-repo-index ` (option :ref:`here `), :ref:`resolve-locations ` (option :ref:`here `) and :ref:`query-repo ` (option :ref:`here `), allowing the user to specify which version comparison algorithm is used. -- Added the ``--search-strat`` option to :ref:`resolve-locations +* Added the ``--search-strat`` option to :ref:`resolve-locations `, allowing users to select breadth first search or depth first search during resolution -- Added the :ref:`matches ` operator (``<>REGEX``) which +* Added the :ref:`matches ` operator (``<>REGEX``) which matches a version against a regex -- Added the :ref:`in-range ` operator (``=>V``) which +* Added the :ref:`in-range ` operator (``=>V``) which matches a version against a certain range of indexes -- Added the ability to specify ``--present-package`` multiple times using the +* Added the ability to specify ``--present-package`` multiple times using the same package name, but different versions. This is useful for when the ``:conflict-strat`` is set to ``inclusive``. -- Added tests testing to make sure that unsuccessful runs generate the proper +* Added tests testing to make sure that unsuccessful runs generate the proper error messages. Changed +++++++ -- Reorganized the unit tests. -- Alphabetized the options for ``generate-card``. -- Alphabetized the options for ``generate-repo-index``. +* Reorganized the unit tests. +* Alphabetized the options for ``generate-card``. +* Alphabetized the options for ``generate-repo-index``. Fixed +++++ -- Fixed bug wherein if the conflict strategy is set to ``:inclusive`` +* Fixed bug wherein if the conflict strategy is set to ``:inclusive`` and a package satisfying a requirement is already found or present, it is used instead of finding a new one. -- Fixed CLI of :ref:`display-config ` so that +* Fixed CLI of :ref:`display-config ` so that it actually works as advertised, LOLZ -- Fixed the CLI output of ``--help`` so that default values +* Fixed the CLI output of ``--help`` so that default values of options are shown again :) -- Refreshed the CLI output of ``--help`` for all the subcommands +* Refreshed the CLI output of ``--help`` for all the subcommands as posted in the docs `1.7.0`_ @@ -274,29 +282,29 @@ Fixed Added +++++ -- Added ``--option-pack``, the ability to :ref:`specify multiple +* Added ``--option-pack``, the ability to :ref:`specify multiple options at once ` Fixed +++++ -- Fixed how default options work, they no longer override stuff +* Fixed how default options work, they no longer override stuff found in the config file (ouch) -- Fixed output of printed warning when configuration file is not used +* Fixed output of printed warning when configuration file is not used `1.6.0`_ -------- Added +++++ -- Formatted docs better on the front page for PDF purposes -- Add ability to use any (long) option on the command line in +* Formatted docs better on the front page for PDF purposes +* Add ability to use any (long) option on the command line in :ref:`display-config ` Improved ++++++++ -- Memoized core Degasolv package system repository function (should +* Memoized core Degasolv package system repository function (should speed the resolver up a bit) -- Changed apt reop function from filtering a list to lookup in a map, +* Changed apt reop function from filtering a list to lookup in a map, increasing its speed `1.5.1`_ @@ -305,7 +313,7 @@ Improved Added +++++ -- In just ~15 seconds, it slurps in a rather large apt repository +* In just ~15 seconds, it slurps in a rather large apt repository Packages.gz file. In another ~45 seconds, it resolves the ubuntu-desktop package, spitting out a grand total of 797 packages with their locations. @@ -313,7 +321,7 @@ Added Fixed +++++ -- While using the apt data and package system to profile Degasolv, I +* While using the apt data and package system to profile Degasolv, I found some rather nasty bugs. This release fixes them. This tool is now ready for prime time. @@ -322,7 +330,7 @@ Fixed Added +++++ -- Added the ``--disable-alternatives`` :ref:`option +* Added the ``--disable-alternatives`` :ref:`option ` and the ``--enable-alternatives`` :ref:`option ` for debugging purposes. @@ -332,7 +340,7 @@ Added Added +++++ -- Added the ``--present-package`` :ref:`option ` and +* Added the ``--present-package`` :ref:`option ` and the ``--package-system`` :ref:`option ` to the :ref:`resolve-locations ` subcommand. This was so that Degasolv could be profiled using apt package repos @@ -344,7 +352,7 @@ Added Added +++++ -- Add standard input as a file type. All options which take a file name may now +* Add standard input as a file type. All options which take a file name may now have ``-`` given as the filename, to specify that standard in should be used. `1.2.0`_ @@ -353,7 +361,7 @@ Added Added +++++ -- Added the ability to specify multiple configuration files, +* Added the ability to specify multiple configuration files, thus allowing for site-wide configuration. `1.1.0`_ @@ -362,15 +370,15 @@ Added Added +++++ -- Added the ``--conflict-strat`` :ref:`option ` to the +* Added the ``--conflict-strat`` :ref:`option ` to the :ref:`resolve-locations ` subcommand. -- Added docs and tests. +* Added docs and tests. 1.0.2 ----- -- This isn't the first release, but for the purposes of these docs, it is :D +* This isn't the first release, but for the purposes of these docs, it is :D .. _Unreleased: https://github.com/djhaskin987/degasolv/compare/2.1.0...HEAD .. _2.1.0: https://github.com/djhaskin987/degasolv/compare/2.0.0...2.1.0 diff --git a/docs/command-reference.rst b/docs/command-reference.rst index 2e04275..33d6ab7 100644 --- a/docs/command-reference.rst +++ b/docs/command-reference.rst @@ -84,30 +84,91 @@ This is true for options of Degasolv and options for any of its subcommands. Explanation of Options ++++++++++++++++++++++ +There are lots of options to degasolv and a few ways in which to specify them. +This section details the ways by which they should be specified. + +Global Options +************** + Degasolv parses global options before it parses subcommands or the options for subcommands; therefore, global options need to be specified first. If any option, whether global or for a subcommand is given incorrectly, the program exits with a return code of 1. +.. _environment-variables: + +Environment Variables +********************* + +Every option in Degasolv has a corresponding environment variable which, if +set, will be consulted for the value of that option. Each option in +this document will have its corresponding environment variable listed next +to it. + +* Options which take a boolean value must be specified as ``true` or ``false`, + as in ``export DEGASOLV_ALTERNATIVES=true``. + +* Options which take a list will be specified as a single string of values + separated by the caret (``^``) character, as in + ``export DEGASOLV_REQUIREMENTS=a^b^c``. + +* The ``:meta`` option is the only option that takes a map or dictionary of + values. In this option, keys and values are separated by the equals sign + (``=``) and the list of key/value pairs are also separated by the caret + character, as in ``k=v^k=v^k=v...`` + +.. note:: The environment variables and their formatting will be + listed for the options of all the subcommands in this document; + however, **environment variables can only be used with Degasolv version + 2.2.0 or greater.** This point bears special emphasis. Lots of config + options say they were released in earlier versions. This is true; however, + the only format of config file available for use was the EDN config file + type before version 1.12.0 of Degasolv. + Using Configuration Files ************************* Configuration files may be specified at the command line before specifying any -subcommands. The config file structure is designed so that any command-line -option may be set in the config file instead, and vice versa. +subcommands, or in the ``DEGASOLV_CONFIG_FILES`` and/or the +``DEGASOLV_JSON_CONFIG_FILES`` environemnt. The config file structure is +designed so that any command-line option may be set in the config file instead, +and vice versa. More information can be found at `edn-config`_ and +`json-config`_ below. In addition, config files may be specified either in the EDN format or JSON format. Multiple config files may be specified. "Mixing and matching" of JSON -and EDN config files is supported. +and EDN config files is supported. For more information, see the +`Multiple Configuration Files`_ section. + +.. _gathering-options: + +How Options are Gathered +************************ + +First, the ``DEGASOLV_CONFIG_FILES`` and ``DEGASOLV_JSON_CONFIG_FILES`` +environment variables are consulted to find any configuration files. + +Next, the options in the configuration files are consulted and are merged onto +each other in the order given in those variables, first EDN files and then JSON +files. The last config file encountered "wins" for any given key for which +multiple files specify a value. + +These options are then added to and overridden by any values in environment +variables, and finally added to and overridden by any values found +by consulting the command line options. + +.. _edn-config: Basic EDN Configuration Usage -############################# +***************************** +-----------------------------+---------------------------------------+ | Short option | ``-c FILE`` | +-----------------------------+---------------------------------------+ | Long option | ``--config-file FILE`` | +-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_CONFIG_FILES=f1^f2^f3`` | ++-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -173,18 +234,18 @@ With this command:: resolve-locations \ [...] - - .. _json-config: Basic JSON Configuration Usage -############################## +****************************** +-----------------------------+---------------------------------------+ | Short option | ``-j FILE`` | +-----------------------------+---------------------------------------+ | Long option | ``--json-config FILE`` | +-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_JSON_CONFIG_FILES=f1^f2`` | ++-----------------------------+---------------------------------------+ | Version introduced | 1.12.0 | +-----------------------------+---------------------------------------+ @@ -235,33 +296,57 @@ The command to use the above JSON config file would look like this:: resolve-locations \ [...] +.. _Multiple Configuration Files: + Using Multiple Configuration Files -################################## +********************************** As of version 1.2.0, the ``--config-file`` option may be specified multiple times. As of version 1.12.0, the ``--json-config`` option may also be -specified, and it too may be multiple times. +specified, and it too may be multiple times. As of version 2.2.0, +configuration files can be specified using the ``DEGASOLV_CONFIG_FILES`` +and ``DEGASOLV_JSON_CONFIG_FILES`` environment variables. Degasolv processes JSON config files together with EDN config files. Each configuration file specified will get its configuration merged into the previously specified configuration files, whether those -files be EDN or JSON. If both configuration files contain the same option, the -option specified in the latter specified configuration file will be used. +files be EDN or JSON. The exception is for environment variables; +the EDN files specified in the environment will be consulted first, followed by +the JSON config files specified in the environment, followed by any +configuration files on the command line whether JSON or EDN. If both +configuration files contain the same option, the option specified in the latter +specified configuration file will be used. .. _config files section: As an example, consider the following `display-config command`_:: + DEGASOLV_JSON_CONFIG_FILES="$PWD/y.json" \ + DEGASOLV_CONFIG_FILES="$PWD/x.edn" \ degasolv \ --config-file "$PWD/a.edn" \ --json-config "$PWD/j.json" \ --config-file "$PWD/b.edn" \ display-config -If this is the contents of the file ``a.edn``:: +If this is the contents of the file ``x.edn``:: + + { + :conflict-strat "inclusive" + :error-format false + } + +And this were the contents of the file ``y.json``:: + + { + "conflict-strat": "prioritized", + "error-format": true + } + +And this is the contents of the file ``a.edn``:: { - :index-strat "priority" + :index-strat "prioritized" :repositories ["https://example.com/repo1/"] :id "a" :version "1.0.0" @@ -270,6 +355,7 @@ If this is the contents of the file ``a.edn``:: And this were the contents of ``j.json``:: { + "id": "j", "alternatives": false, "requirements": ["x", "y"] } @@ -277,7 +363,6 @@ And this were the contents of ``j.json``:: And this were the contents of ``b.edn``:: { - :conflict-strat "exclusive" :repositories ["https://example.com/repo2/"] :id "b" :version "2.0.0" @@ -287,14 +372,15 @@ And this were the contents of ``b.edn``:: Then the output of the above command would look like this:: { - :alternatives false, - :index-strat "priority", - :repositories ["https://example.com/repo2/"], - :id "b", - :version "2.0.0", - :conflict-strat "exclusive", + :alternatives false + :error-format true + :index-strat "priority" + :repositories ["https://example.com/repo2/"] + :id "b" + :version "2.0.0" + :conflict-strat "prioritized" :requirements [] - :arguments ["display-config"], + :arguments ["display-config"] } .. note:: The JSON config file keys and their formatting will be @@ -305,10 +391,30 @@ Then the output of the above command would look like this:: format of config file available for use was the EDN config file type before version 1.12.0 of Degasolv. + +.. _default-configuration-files: + +Default Configuration Files +*************************** + +All previous versions prior to 2.2.0 of degasolv will look for a file called +``./degasolv.edn`` if no other config file was specified. + +As of version 2.2.0, If no configuration files are specified, they will be +looked for in the following locations, if they exist, as if they were specified +in the following order on the command line: + +1. ``${AppData}/degasolv/config.edn`` +2. ``${AppData}/degasolv/config.json`` +3. ``${HOME}/.degasolv.edn`` +4. ``${HOME}/.degasolv.json`` +5. ``./degasolv.edn`` +6. ``./degasolv.json`` + .. _site-wide: Using Site-Wide Configuration Files -################################### +*********************************** The merging of config files, together with the interesting fact that config files may be specified via HTTP/HTTPS URLs, @@ -327,6 +433,21 @@ build-specific config file, as in this example:: --config-file "./degasolv.edn" \ generate-card +Also remember that config files can be specified as environment variables. For +example, the above example would look like this, if environment variables +were used:: + + export DEGASOLV_CONFIG_FILES="https://nas.example.com/degasolv/site.edn^./degasolv.edn" + degasolv \ + generate-card + +Here is a version of that example that uses JSON instead:: + + + export DEGASOLV_JSON_CONFIG_FILES="https://nas.example.com/degasolv/site.json^./degasolv.json" + degasolv \ + generate-card + .. _option-pack: .. _option pack: @@ -338,38 +459,52 @@ Option Packs +-----------------------------+---------------------------------------+ | Long option | ``--option-pack PACK`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:option-packs ["PACK1",...]`` | +| EDN config file key | ``:option-packs ["PACK1",...]`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"option-packs": ["PACK1",...],`` | +| JSON config file key | ``"option-packs": ["PACK1",...],`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_OPTION_PACKS="P1^P2^..."`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.7.0 | +-----------------------------+---------------------------------------+ -Specify one or more option packs. +Specify one or more option packs. The commandline version +of this option may be specified multiple times. Degasolv ships with several "option packs", each of which imply several Degasolv options at once. When an option pack is specified, Degasolv looks up which option pack is used and what options are -implied by using it. More than one option pack may be specified. If -option packs are specified both on the command line and in the config -file, the option packs on the command line are used and the ones in -the config file are ignored. +implied by using it. More than one option pack may be specified. + +Prior to version 2.2.0, If option packs were specified both on the command line +and in the config file, the option packs on the command line are used and the +ones in the config file were ignored. + +As of version 2.2.0, Option packs are "expanded" into +the options they imply on the level in which they are specified, where in a +particular configuration file, in the environment, or on the commandline. Then +options are merged according to the usual rules -- first configuration files +are merged (see `Multiple Configuration Files`_ on how they are merged), then +environment variables, and finally commandline options. The following option packs are supported in the current version: - - ``v1``: Added as of version 2.0.0 . Implies - ``--list-strat as-set`` and ``--disable-error-format``. This - pack was added to help support legacy deployments of Degasolv. - It should be noted that to achieve full compatibility with Degasolv - version 1, the argument ``--version-comparison maven`` should be used - as well as this option pack. It could not be included in the option - pack due to complications with the version comparison option and its - relationship to how the ``--package-system`` option is affected by it. - - ``multi-version-mode``: Added as of version 1.7.0 . Implies - ``--conflict-strat inclusive``, - ``--resolve-strat fast``, and ``--disable-alternatives``. - - ``firstfound-version-mode``: Added as of version 1.7.0 . Implies - ``--conflict-strat prioritized``, - ``--resolve-strat fast``, and ``--disable-alternatives``. + +* ``v1``: Added as of version 2.0.0 . Implies ``--list-strat as-set`` and + ``--disable-error-format``. This pack was added to help support legacy + deployments of Degasolv. It should be noted that to achieve full + compatibility with Degasolv version 1, the argument ``--version-comparison + maven`` should be used as well as this option pack. It could not be included + in the option pack due to complications with the version comparison option + and its relationship to how the ``--package-system`` option is affected by + it. + +* ``multi-version-mode``: Added as of version 1.7.0 . Implies + ``--conflict-strat inclusive``, ``--resolve-strat fast``, and + ``--disable-alternatives``. + +* ``firstfound-version-mode``: Added as of version 1.7.0 . Implies + ``--conflict-strat prioritized``, ``--resolve-strat fast``, and + ``--disable-alternatives``. Print the Help Page ******************* @@ -502,9 +637,11 @@ Specify Location of the Card File +-----------------------------+---------------------------------------+ | Long option | ``--card-file FILE`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:card-file "FILE"`` | +| EDN config file key | ``:card-file "FILE"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"card-file": "FILE"`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"card-file": ["FILE",...],`` | +| Environment variable | ``DEGASOLV_CARD_FILE="FILE"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -525,9 +662,11 @@ Specify the ID (Name) of the Package +-----------------------------+---------------------------------------+ | Long option | ``--id ID`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:id "ID"`` | +| EDN config file key | ``:id "ID"`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"id": "ID",`` | +| JSON config file key | ``"id": "ID",`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_ID="ID"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -545,9 +684,11 @@ Specify the Location of the Package +-----------------------------+---------------------------------------+ | Long option | ``--location LOCATION`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:location "LOCATION"`` | +| EDN config file key | ``:location "LOCATION"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"location": "LOCATION",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"location": "LOCATION",`` | +| Environment variable | ``DEGASOLV_LOCATION="LOCATION"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -567,9 +708,11 @@ Specify Additional Metadata for a Package +-----------------------------+---------------------------------------+ | Long option | ``--meta K=V`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:meta {:key1 "value1" ...}`` | +| EDN config file key | ``:meta {:key1 "value1" ...}`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"meta": {"key1": "value1", ...},`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"meta": {"key1": "value1", ...},`` | +| Environment variable | ``DEGASOLV_META="k1=v1^k2=v2..."`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.11.0 | +-----------------------------+---------------------------------------+ @@ -620,9 +763,11 @@ Specify a Requirement for a Package +-----------------------------+---------------------------------------+ | Long option | ``--requirement REQ`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:requirements ["REQ1", ...]`` | +| EDN config file key | ``:requirements ["REQ1", ...]`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"requirements": ["REQ1", ...],`` | +| JSON config file key | ``"requirements": ["REQ1", ...],`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_REQUIREMENTS="r1^r2..."`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -640,9 +785,11 @@ Specify a Version for a Package +-----------------------------+---------------------------------------+ | Long option | ``--version VERSION`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:version "VERSION"`` | +| EDN config file key | ``:version "VERSION"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"version": "VERSION",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"version": "VERSION",`` | +| Environment variable | ``DEGASOLV_VERSION="VERSION"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -710,9 +857,11 @@ Specify the Repo Search Directory +-----------------------------+---------------------------------------+ | Long option | ``--search-directory DIR`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:search-directory "DIR"`` | +| EDN config file key | ``:search-directory "DIR"`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"search-directory": "DIR",`` | +| JSON config file key | ``"search-directory": "DIR",`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_SEARCH_DIRECTORY="DIR"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -730,9 +879,11 @@ Specify the Repo Index File +-----------------------------+---------------------------------------+ | Long option | ``--index-file FILE`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:index-file "FILE"`` | +| EDN config file key | ``:index-file "FILE"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"index-file": "FILE",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"index-file": "FILE",`` | +| Environment variable | ``DEGASOLV_INDEX_FILE="FILE"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -750,9 +901,11 @@ Specify the Index Sort Order +-----------------------------+---------------------------------------+ | Long option | ``--index-sort-order ORDER`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:index-sort-order "ORDER"`` | +| EDN config file key | ``:index-sort-order "ORDER"`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"index-sort-order": "ORDER",`` | +| JSON config file key | ``"index-sort-order": "ORDER",`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_INDEX_SORT_ORDER="ORDER"`` | +-----------------------------+---------------------------------------+ | Version introduced | 2.1.0 | +-----------------------------+---------------------------------------+ @@ -789,9 +942,11 @@ Specify the Version Comparison Algorithm +-----------------------------+---------------------------------------+ | Long option | ``--version-comparison CMP`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:version-comparison "CMP"`` | +| EDN config file key | ``:version-comparison "CMP"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"version-comparison": "CMP",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"version-comparison": "CMP",`` | +| Environment variable | ``DEGASOLV_VERSION_COMPARISON="CMP"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.8.0 | +-----------------------------+---------------------------------------+ @@ -827,9 +982,11 @@ Add to an Existing Repository Index +-----------------------------+---------------------------------------+ | Long option | ``--add-to INDEX`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:add-to "INDEX"`` | +| EDN config file key | ``:add-to "INDEX"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"add-to": "INDEX",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"add-to": "INDEX",`` | +| Environment variable | ``DEGASOLV_ADD_TO="INDEX"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -966,9 +1123,11 @@ Enable the Use of Alternatives +-----------------------------+---------------------------------------+ | Long option | ``--enable-alternatives`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:alternatives true`` | +| EDN config file key | ``:alternatives true`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"alternatives": true,`` | +| JSON config file key | ``"alternatives": true,`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_ALTERNATIVES="true"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.5.0 | +-----------------------------+---------------------------------------+ @@ -991,9 +1150,11 @@ Disable the Use of Alternatives +-----------------------------+---------------------------------------+ | Long option | ``--disable-alternatives`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:alternatives false`` | +| EDN config file key | ``:alternatives false`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"alternatives": false,`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"alternatives": false,`` | +| Environment variable | ``DEGASOLV_ALTERNATIVES="false"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.5.0 | +-----------------------------+---------------------------------------+ @@ -1024,9 +1185,11 @@ Specify Solution Search Strategy +-----------------------------+---------------------------------------+ | Long option | ``--search-strat STRAT`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:search-strat "STRAT"`` | +| EDN config file key | ``:search-strat "STRAT"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"search-strat": "STRAT",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"search-strat": "STRAT",`` | +| Environment variable | ``DEGASOLV_SEARCH_STRAT="STRAT"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.8.0 | +-----------------------------+---------------------------------------+ @@ -1048,9 +1211,11 @@ Specify Conflict Strategy +-----------------------------+---------------------------------------+ | Long option | ``--conflict-strat STRAT`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:conflict-strat "STRAT"`` | +| EDN config file key | ``:conflict-strat "STRAT"`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"conflict-strat": "STRAT",`` | +| JSON config file key | ``"conflict-strat": "STRAT",`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_CONFLICT_STRAT="STRAT"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.1.0 | +-----------------------------+---------------------------------------+ @@ -1113,9 +1278,11 @@ Specify List Strategy +-----------------------------+---------------------------------------+ | Long option | ``--list-strat STRAT`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:list-strat "STRAT"`` | +| EDN config file key | ``:list-strat "STRAT"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"list-strat": "STRAT",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"list-strat": "STRAT",`` | +| Environment variable | ``DEGASOLV_LIST_STRAT="STRAT"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.12.0 | +-----------------------------+---------------------------------------+ @@ -1164,7 +1331,7 @@ the following rules: sheep==1.0 @ http://example.com/repo/sheep-1.0.zip steel==1.0 @ http://example.com/repo/steel-1.0.zip - It is worth noting that commandline arguments are listed in + It is worth noting that command line arguments are listed in reverse order. Thus, generating a card file with arguments ``-r wool -r wood -r sheep`` would yield a list that looks like this:: @@ -1188,9 +1355,11 @@ Enable Error Output Format +-----------------------------+---------------------------------------+ | Long option | ``--enable-error-format`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:error-format true`` | +| EDN config file key | ``:error-format true`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"error-format": true,`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"error-format": true,`` | +| Environment variable | ``DEGASOLV_ERROR_FORMAT="true"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.12.0 | +-----------------------------+---------------------------------------+ @@ -1227,9 +1396,11 @@ Disable Error Output Format +-----------------------------+---------------------------------------+ | Long option | ``--disable-error-format`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:error-format false`` | +| EDN config file key | ``:error-format false`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"error-format": false,`` | +| JSON config file key | ``"error-format": false,`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_ERROR_FORMAT="false"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.12.0 | +-----------------------------+---------------------------------------+ @@ -1246,9 +1417,11 @@ Specify Output Format +-----------------------------+---------------------------------------+ | Long option | ``--output-format FORMAT`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:output-format "FORMAT"`` | +| EDN config file key | ``:output-format "FORMAT"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"output-format": "FORMAT",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"output-format": "FORMAT",`` | +| Environment variable | ``DEGASOLV_OUTPUT_FORMAT="FORMAT"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.10.0; EDN introduced 1.11.0 | +-----------------------------+---------------------------------------+ @@ -1444,9 +1617,11 @@ Specify that a Package is Already Present +-----------------------------+----------------------------------------+ | Long option | ``--present-package PKG`` | +-----------------------------+----------------------------------------+ -| EDN Config file key | ``:present-packages ["PKG1", ...]`` | +| EDN config file key | ``:present-packages ["PKG1", ...]`` | +-----------------------------+----------------------------------------+ -| JSON Config file key | ``"present-packages": ["PKG1", ...],`` | +| JSON config file key | ``"present-packages": ["PKG1", ...],`` | ++-----------------------------+----------------------------------------+ +| Environment variable | ``DEGASOLV_PRESENT_PACKAGES="P1^..."`` | +-----------------------------+----------------------------------------+ | Version introduced | 1.4.0 | +-----------------------------+----------------------------------------+ @@ -1484,9 +1659,11 @@ Specify a Requirement +-----------------------------+---------------------------------------+ | Long option | ``--requirement REQ`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:requirements ["REQ1", ...]`` | +| EDN config file key | ``:requirements ["REQ1", ...]`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"requirements": ["REQ1", ...],`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"requirements": ["REQ1", ...],`` | +| Environment variable | ``DEGASOLV_REQUIREMENTS="R1^R2^..."`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -1514,9 +1691,11 @@ Specify a Repository to Search +-----------------------------+---------------------------------------+ | Long option | ``--repository INDEX`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:repositories ["INDEX1", ...]`` | +| EDN config file key | ``:repositories ["INDEX1", ...]`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"repositories": ["INDEX1", ...],`` | +| JSON config file key | ``"repositories": ["INDEX1", ...],`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_REPOSITORIES="I1^I2^..."`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -1557,9 +1736,11 @@ Specify a Resolution Strategy +-----------------------------+---------------------------------------+ | Long option | ``--resolve-strat STRAT`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:resolve-strat "STRAT"`` | +| EDN config file key | ``:resolve-strat "STRAT"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"resolve-strat": "STRAT",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"resolve-strat": "STRAT",`` | +| Environment variable | ``DEGASOLV_RESOLVE_STRAT="I1^..."`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -1589,9 +1770,11 @@ Specify an Index Strategy +-----------------------------+---------------------------------------+ | Long option | ``--index-strat STRAT`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:index-strat "STRAT"`` | +| EDN config file key | ``:index-strat "STRAT"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"index-strat": "STRAT",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"index-strat": "STRAT",`` | +| Environment variable | ``DEGASOLV_INDEX_STRAT="STRAT"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -1645,9 +1828,11 @@ Specify a Package System +-----------------------------+---------------------------------------+ | Long option | ``--package-system SYS`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:package-system "SYS"`` | +| EDN config file key | ``:package-system "SYS"`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"package-system": "SYS",`` | +| JSON config file key | ``"package-system": "SYS",`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_PACKAGE_SYSTEM="SYS"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.4.0 | +-----------------------------+---------------------------------------+ @@ -1774,9 +1959,11 @@ Specify Subproc Package System Output Format +-----------------------------+----------------------------------------+ | Long option | ``--subproc-output-format FORMAT`` | +-----------------------------+----------------------------------------+ -| EDN Config file key | ``:subproc-output-format "FORMAT"`` | +| EDN config file key | ``:subproc-output-format "FORMAT"`` | ++-----------------------------+----------------------------------------+ +| JSON config file key | ``"subproc-output-format": "FORMAT",`` | +-----------------------------+----------------------------------------+ -| JSON Config file key | ``"subproc-output-format": "FORMAT",`` | +| Environment variable | ``DEGASOLV_SUBPROC_OUTPUT_FORMAT="F"`` | +-----------------------------+----------------------------------------+ | Version introduced | 1.12.0 | +-----------------------------+----------------------------------------+ @@ -1797,9 +1984,11 @@ Specify the Version Comparison Algorithm +-----------------------------+---------------------------------------+ | Long option | ``--version-comparison CMP`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:version-comparison "CMP"`` | +| EDN config file key | ``:version-comparison "CMP"`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"version-comparison": "CMP",`` | +| JSON config file key | ``"version-comparison": "CMP",`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_VERSION_COMPARISON="CMP"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.8.0 | +-----------------------------+---------------------------------------+ @@ -1837,9 +2026,11 @@ Specify Subproc Package System Executable +-----------------------------+---------------------------------------+ | Long option | ``--subproc-exe PATH`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:subproc-exe "PATH"`` | +| EDN config file key | ``:subproc-exe "PATH"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"subproc-exe": "PATH",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"subproc-exe": "PATH",`` | +| Environment variable | ``DEGASOLV_SUBPROC_EXE="PATH"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.12.0 | +-----------------------------+---------------------------------------+ @@ -1905,9 +2096,11 @@ Enable Error Output Format +-----------------------------+---------------------------------------+ | Long option | ``--enable-error-format`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:error-format true`` | +| EDN config file key | ``:error-format true`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"error-format": true,`` | +| JSON config file key | ``"error-format": true,`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_ERROR_FORMAT="true"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.12.0 | +-----------------------------+---------------------------------------+ @@ -1946,9 +2139,11 @@ Disable Error Output Format +-----------------------------+---------------------------------------+ | Long option | ``--disable-error-format`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:error-format false`` | +| EDN config file key | ``:error-format false`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"error-format": false,`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"error-format": false,`` | +| Environment variable | ``DEGASOLV_ERROR_FORMAT="false"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.12.0 | +-----------------------------+---------------------------------------+ @@ -1965,9 +2160,11 @@ Specify Output Format +-----------------------------+---------------------------------------+ | Long option | ``--output-format FORMAT`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:output-format "FORMAT"`` | +| EDN config file key | ``:output-format "FORMAT"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"output-format": "FORMAT"`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"output-format": "FORMAT"`` | +| Environment variable | ``DEGASOLV_OUTPUT_FORMAT="FORMAT"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.11.0 | +-----------------------------+---------------------------------------+ @@ -2017,9 +2214,11 @@ Specify a Repository to Search +-----------------------------+---------------------------------------+ | Long option | ``--repository INDEX`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:repositories ["INDEX1", ...]`` | +| EDN config file key | ``:repositories ["INDEX1", ...]`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"repositories": ["INDEX1", ...],`` | +| JSON config file key | ``"repositories": ["INDEX1", ...],`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_REPOSITORIES="I1^I2^..."`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -2038,9 +2237,11 @@ Specify an Index Strategy +-----------------------------+---------------------------------------+ | Long option | ``--index-strat STRAT`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:index-strat "STRAT"`` | +| EDN config file key | ``:index-strat "STRAT"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"index-strat": "STRAT",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"index-strat": "STRAT",`` | +| Environment variable | ``DEGASOLV_INDEX_STRAT="STRAT"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.0.2 | +-----------------------------+---------------------------------------+ @@ -2052,11 +2253,19 @@ queries. See that option's explanation for more information. Specify a Package System ************************ -+--------------+---------------------------+-----------------------------------+ -| Short option | Long option | Config File Key | -+--------------+---------------------------+-----------------------------------+ -| ``-t SYS`` | ``--package-system SYS`` | ``:package-system "SYS"`` | -+--------------+---------------------------+-----------------------------------+ ++-----------------------------+---------------------------------------+ +| Short option | ``-t SYS`` | ++-----------------------------+---------------------------------------+ +| Long option | ``--package-system SYS`` | ++-----------------------------+---------------------------------------+ +| EDN config file key | ``:package-system "SYS"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"package-system": "SYS",`` | ++-----------------------------+---------------------------------------+ +| Environment variable | ``DEGASOLV_PACKAGE_SYSTEM="SYS"`` | ++-----------------------------+---------------------------------------+ +| Version introduced | 1.4.0 | ++-----------------------------+---------------------------------------+ This option works exactly the same as the `package system`_ option for the ``resolve-locations`` command, except that it is used for simple @@ -2072,9 +2281,11 @@ Specify the Version Comparison Algorithm +-----------------------------+---------------------------------------+ | Long option | ``--version-comparison CMP`` | +-----------------------------+---------------------------------------+ -| EDN Config file key | ``:version-comparison "CMP"`` | +| EDN config file key | ``:version-comparison "CMP"`` | ++-----------------------------+---------------------------------------+ +| JSON config file key | ``"version-comparison": "CMP",`` | +-----------------------------+---------------------------------------+ -| JSON Config file key | ``"version-comparison": "CMP",`` | +| Environment variable | ``DEGASOLV_VERSION_COMPARISON="CMP"`` | +-----------------------------+---------------------------------------+ | Version introduced | 1.8.0 | +-----------------------------+---------------------------------------+ From a86ee3d7e4389cfafd55732aac1c6a326d6bba3c Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Fri, 8 Nov 2019 21:18:59 -0700 Subject: [PATCH 11/31] Add smarter slurp --- project.clj | 1 + src/degasolv/util.clj | 25 +++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index bc4e9c9..8092641 100644 --- a/project.clj +++ b/project.clj @@ -13,6 +13,7 @@ ;;[crouton "0.1.2"] ;;[ring/ring-codec "1.1.2"] ;;[ + [clj-http "3.10.0"] [com.velisco/tagged "0.5.0"] [org.clojure/clojure "1.10.1"] [org.clojure/tools.cli "0.3.5"] diff --git a/src/degasolv/util.clj b/src/degasolv/util.clj index fbe4e12..cad594a 100644 --- a/src/degasolv/util.clj +++ b/src/degasolv/util.clj @@ -1,7 +1,8 @@ (ns degasolv.util (:require [clojure.java.io :as io] [clojure.java.io :as io] - [clojure.pprint :as pprint])) + [clojure.pprint :as pprint] + [clj-http.client :as client])) ; Deprecated, we should just use update-in (defn assoc-conj @@ -9,12 +10,32 @@ (update-in mp [k] conj v)) ; UTF-8 by default :) -(defn default-slurp [loc] +(defn base-slurp [loc] (let [input (if (= loc "-") *in* loc)] (clojure.core/slurp input :encoding "UTF-8"))) +(defn default-slurp [resource] + (if-let [[whole-thing protocol auth-stuff rest-of-it] + (re-matches #"(https?://)([^@]+)@(.+)" resource)] + (if-let [[_ username password] + (re-matches #"([^:]+):([^:]+)" auth-stuff)] + (client/get (str + protocol + rest-of-it) + {:basic-auth [(java.net.URLDecoder/decode username) + (java.net.URLDecoder/decode password)]}) + (client/get (str + protocol + rest-of-it) + {:headers + {"Authorization" (str "Bearer " + (java.net.URLDecoder/decode + auth-stuff))}})) + (base-slurp resource))) + + (defn default-spit [loc stuff] (clojure.core/spit loc (pr-str stuff) :encoding "UTF-8")) From 328ddb78ff051272c7fc16335ffa9cf0cdf05ad2 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sat, 9 Nov 2019 16:46:50 -0700 Subject: [PATCH 12/31] make a change but it's making things slow --- src/degasolv/pkgsys/core.clj | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/degasolv/pkgsys/core.clj b/src/degasolv/pkgsys/core.clj index 9c785a6..1e9691d 100644 --- a/src/degasolv/pkgsys/core.clj +++ b/src/degasolv/pkgsys/core.clj @@ -49,11 +49,14 @@ (fn merg [c v] (update-in c [(:id v)] conj v)) initial-repository - (map - read-card! + (->> search-directory + (io/file) + (file-seq) (filter #(and (.isFile ^java.io.File (io/file %)) - (= "dscard" (st/replace % #"[^.]*[.]" ""))) - (file-seq (io/file search-directory)))))))))) + (= "dscard" (st/replace % #"[^.]*[.]" "")))) + (map (fn [^java.io.File f] (.getAbsolutePath f))) + (map read-card!) + ))))))) (defn slurp-degasolv-repo [url] From 2f1a98fbb12f83c1da0eaf7675177fff45b190b2 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Mon, 11 Nov 2019 17:23:59 -0700 Subject: [PATCH 13/31] Fix the problem --- src/degasolv/pkgsys/core.clj | 3 ++- src/degasolv/util.clj | 4 ++-- test/resources/scripts/test-http-auth.sh | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 test/resources/scripts/test-http-auth.sh diff --git a/src/degasolv/pkgsys/core.clj b/src/degasolv/pkgsys/core.clj index 1e9691d..ea7b371 100644 --- a/src/degasolv/pkgsys/core.clj +++ b/src/degasolv/pkgsys/core.clj @@ -26,7 +26,8 @@ card-data))) (defn generate-repo-index! - [search-directory + [ + search-directory index-file add-to sortindex diff --git a/src/degasolv/util.clj b/src/degasolv/util.clj index cad594a..555ffef 100644 --- a/src/degasolv/util.clj +++ b/src/degasolv/util.clj @@ -19,7 +19,7 @@ (defn default-slurp [resource] (if-let [[whole-thing protocol auth-stuff rest-of-it] (re-matches #"(https?://)([^@]+)@(.+)" resource)] - (if-let [[_ username password] + (:body (if-let [[_ username password] (re-matches #"([^:]+):([^:]+)" auth-stuff)] (client/get (str protocol @@ -32,7 +32,7 @@ {:headers {"Authorization" (str "Bearer " (java.net.URLDecoder/decode - auth-stuff))}})) + auth-stuff))}}))) (base-slurp resource))) diff --git a/test/resources/scripts/test-http-auth.sh b/test/resources/scripts/test-http-auth.sh new file mode 100644 index 0000000..57d76ec --- /dev/null +++ b/test/resources/scripts/test-http-auth.sh @@ -0,0 +1 @@ +lein run -- query-repo --repository http://abc:123@k70rg.mocklab.io/thing/8 --query d From be38f9651ad553a0c09deadccb97016e45a45122 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Mon, 11 Nov 2019 22:21:30 -0700 Subject: [PATCH 14/31] trying to get something going --- project.clj | 1 + test/degasolv/util_funcs_test.clj | 51 ++++++++++++++++++++++++ test/resources/scripts/test-http-auth | 27 +++++++++++++ test/resources/scripts/test-http-auth.sh | 1 - 4 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 test/degasolv/util_funcs_test.clj create mode 100755 test/resources/scripts/test-http-auth delete mode 100644 test/resources/scripts/test-http-auth.sh diff --git a/project.clj b/project.clj index 8092641..94fd847 100644 --- a/project.clj +++ b/project.clj @@ -36,6 +36,7 @@ :profiles { :dev { :dependencies [ + [clj-wiremock "0.3.0"] [pjstadig/humane-test-output "0.9.0"] ;; ordered set for version suggestion [org.flatland/ordered "1.5.7"] diff --git a/test/degasolv/util_funcs_test.clj b/test/degasolv/util_funcs_test.clj new file mode 100644 index 0000000..2ecdd81 --- /dev/null +++ b/test/degasolv/util_funcs_test.clj @@ -0,0 +1,51 @@ +(ns degasolv.util-funcs-test + (:require [clojure.test :refer :all] + [degasolv.util :refer :all] + [clj-wiremock.core :refer :all])) + +(deftest ^:integration-tests + slurp-basic-auth + (let [wiremock-server (server)] + (start wiremock-server) + (stub + {:request + { + :method "GET" + :url "/basic-auth" + :headers { + :Authorization "Basic YWJjOjEyMw==" + } + } + :response + { + :status 200 + :body "boy oh boy" + :headers { + :Content-Type "text/plain" + } + } + }) + (stub + {:request + { + :method "GET" + :url "/query-string" + :queryParameters { + :search "in" + } + } + :response { + :headers { + :Content-Type "text/plain" + } + :status 200 + :body "beef" + } + + }) + (testing "What does query string do" + (is (= "beef" (default-slurp "http://localhost:8080/query-string?in=tact")))) + (testing "Basic authentication" + (is (= "boy oh boy" (default-slurp "http://abc:123@localhost:8080/basic-auth"))) + ) + (stop wiremock-server))) diff --git a/test/resources/scripts/test-http-auth b/test/resources/scripts/test-http-auth new file mode 100755 index 0000000..2a6ae0e --- /dev/null +++ b/test/resources/scripts/test-http-auth @@ -0,0 +1,27 @@ +#!/bin/sh +export POSIXLY_CORRECT=1 +set -x +if [ ! -f './project.clj' -a ! -f './build.boot' ] +then + echo "This script must be run from the root of the project." +fi + +test_home=test/resources/data/test-http-auth + + + +name=$(lein print :name | sed 's|"||g') +version=$(lein print :version | sed 's|"||g') + +java -jar test/resources/data/test-http-auth/wiremock- + +java \ + -Xss16m \ + -jar target/uberjar/${name}-${version}-standalone.jar \ + query-repo \ + --repository "binary-amd64 file://${PWD}/${test_home}/avail /" \ + --package-system apt \ + --query "libc6" + +try_time java \ +lein run -- query-repo --repository http://abc:123@k70rg.mocklab.io/thing/8?fie=ongoodness --query d diff --git a/test/resources/scripts/test-http-auth.sh b/test/resources/scripts/test-http-auth.sh deleted file mode 100644 index 57d76ec..0000000 --- a/test/resources/scripts/test-http-auth.sh +++ /dev/null @@ -1 +0,0 @@ -lein run -- query-repo --repository http://abc:123@k70rg.mocklab.io/thing/8 --query d From b7805193e02812a1a17326b75cfc6672d7edbb27 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Mon, 11 Nov 2019 23:08:22 -0700 Subject: [PATCH 15/31] Add some good tests --- src/degasolv/pkgsys/core.clj | 2 +- test/degasolv/util/core_test.clj | 64 ++++++++++++++++++++++++++++++- test/degasolv/util_funcs_test.clj | 51 ------------------------ 3 files changed, 63 insertions(+), 54 deletions(-) delete mode 100644 test/degasolv/util_funcs_test.clj diff --git a/src/degasolv/pkgsys/core.clj b/src/degasolv/pkgsys/core.clj index ea7b371..26b966c 100644 --- a/src/degasolv/pkgsys/core.clj +++ b/src/degasolv/pkgsys/core.clj @@ -10,7 +10,7 @@ (defn- read-card! [card] - (let [card-data (tag/read-string (default-slurp card)) + (let [card-data (tag/read-string (base-slurp card)) vetted-card-data (s/conform ::r/package card-data)] (if (= vetted-card-data diff --git a/test/degasolv/util/core_test.clj b/test/degasolv/util/core_test.clj index ade0486..8eb26ab 100644 --- a/test/degasolv/util/core_test.clj +++ b/test/degasolv/util/core_test.clj @@ -1,6 +1,66 @@ -(ns degasolv.util.core_test +(ns degasolv.util.core-test (:require [clojure.test :refer :all] - [degasolv.util :refer :all])) + [degasolv.util :refer :all] + [clj-wiremock.core :refer :all])) + +(deftest ^:integration-tests + slurp-auth + (let [wiremock-server (server)] + (start wiremock-server) + (stub + {:request + { + :method "GET" + :url "/unauthenticated" + } + :response + { + :status 200 + :body "bay" + :headers { + :Content-Type "text/plain" + } + }}) + (stub + {:request + { + :method "GET" + :url "/bearer-auth" + :headers { + :Authorization { :equalTo "Bearer deadbeef" } + } + } + :response + { + :status 200 + :body "ay" + :headers { + :Content-Type "text/plain" + } + }}) + (stub + {:request + { + :method "GET" + :url "/basic-auth" + :headers { + :Authorization { :equalTo "Basic YWJjOjEyMw==" } + } + } + :response + { + :status 200 + :body "boy oh boy" + :headers { + :Content-Type "text/plain" + } + } + }) + (testing "Basic authentication" + (is (= "bay" (default-slurp "http://localhost:8080/unauthenticated"))) + (is (= "boy oh boy" (default-slurp "http://abc:123@localhost:8080/basic-auth"))) + (is (= "ay" (default-slurp "http://deadbeef@localhost:8080/bearer-auth"))) + (stop wiremock-server)))) (deftest ^:unit-tests assoc-conj-basic (testing "Add to a blank map" diff --git a/test/degasolv/util_funcs_test.clj b/test/degasolv/util_funcs_test.clj deleted file mode 100644 index 2ecdd81..0000000 --- a/test/degasolv/util_funcs_test.clj +++ /dev/null @@ -1,51 +0,0 @@ -(ns degasolv.util-funcs-test - (:require [clojure.test :refer :all] - [degasolv.util :refer :all] - [clj-wiremock.core :refer :all])) - -(deftest ^:integration-tests - slurp-basic-auth - (let [wiremock-server (server)] - (start wiremock-server) - (stub - {:request - { - :method "GET" - :url "/basic-auth" - :headers { - :Authorization "Basic YWJjOjEyMw==" - } - } - :response - { - :status 200 - :body "boy oh boy" - :headers { - :Content-Type "text/plain" - } - } - }) - (stub - {:request - { - :method "GET" - :url "/query-string" - :queryParameters { - :search "in" - } - } - :response { - :headers { - :Content-Type "text/plain" - } - :status 200 - :body "beef" - } - - }) - (testing "What does query string do" - (is (= "beef" (default-slurp "http://localhost:8080/query-string?in=tact")))) - (testing "Basic authentication" - (is (= "boy oh boy" (default-slurp "http://abc:123@localhost:8080/basic-auth"))) - ) - (stop wiremock-server))) From bf19728bc90a3728ab80bd92172bc8051ceaa16f Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Fri, 15 Nov 2019 08:16:10 -0700 Subject: [PATCH 16/31] Clip in the wall. Close to HTTP authentication --- README.md | 5 +-- docs/changelog.rst | 2 ++ src/degasolv/util.clj | 56 ++++++++++++++++++++------------ test/degasolv/util/core_test.clj | 56 ++++++++++++++++++++++++-------- 4 files changed, 81 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index cef7623..22ef443 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,6 @@ Named for *Degas* the painter, and it's a *Solv*er. ## Download - - Ubuntu: [ ![Download for Ubuntu](https://api.bintray.com/packages/degasolv/ubuntu/degasolv/images/download.svg) ](https://bintray.com/degasolv/ubuntu/degasolv/_latestVersion) CentOS: [ ![Download for CentOS](https://api.bintray.com/packages/degasolv/centos/degasolv/images/download.svg) ](https://bintray.com/degasolv/centos/degasolv/_latestVersion) @@ -28,7 +26,6 @@ tab for more (including the jar file). This quickstart is meant to be illustrative. For ideas on how to use degasolv in real life, have a look at [A Longer Example](http://degasolv.readthedocs.io/en/latest/longer-example.html). - **Given these artifacts**: - `http://example.com/repo/a-1.0.zip` @@ -98,7 +95,7 @@ See [here](http://bit.ly/degasolv2017pres). ## License -Copyright © 2016-2017 Daniel Jay Haskin and others, see the AUTHORS.md file. +Copyright © 2016-2019 Daniel Jay Haskin and others, see the AUTHORS.md file. Distributed under the Eclipse Public License either version 1.0 or (at your option) any later version. diff --git a/docs/changelog.rst b/docs/changelog.rst index 1e0c222..8186b8b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -31,6 +31,8 @@ Added Changed +++++++ +* Removed deprecated, unused function ``degasolv.util.assoc-conj`` + Fixed +++++ diff --git a/src/degasolv/util.clj b/src/degasolv/util.clj index 555ffef..4e477b5 100644 --- a/src/degasolv/util.clj +++ b/src/degasolv/util.clj @@ -4,10 +4,12 @@ [clojure.pprint :as pprint] [clj-http.client :as client])) -; Deprecated, we should just use update-in -(defn assoc-conj - [mp k v] - (update-in mp [k] conj v)) + +(defmacro dbg [body] + `(let [x# ~body] + (println "dbg:" '~body "=" x#) + (flush) + x#)) ; UTF-8 by default :) (defn base-slurp [loc] @@ -17,23 +19,35 @@ (clojure.core/slurp input :encoding "UTF-8"))) (defn default-slurp [resource] - (if-let [[whole-thing protocol auth-stuff rest-of-it] - (re-matches #"(https?://)([^@]+)@(.+)" resource)] - (:body (if-let [[_ username password] - (re-matches #"([^:]+):([^:]+)" auth-stuff)] - (client/get (str - protocol - rest-of-it) - {:basic-auth [(java.net.URLDecoder/decode username) - (java.net.URLDecoder/decode password)]}) - (client/get (str - protocol - rest-of-it) - {:headers - {"Authorization" (str "Bearer " - (java.net.URLDecoder/decode - auth-stuff))}}))) - (base-slurp resource))) + (if (re-matches #"https?://.*" resource) + (if-let [[whole-thing protocol auth-stuff rest-of-it] + (re-matches #"(https?://)([^@]+)@(.+)" resource)] + (:body (if-let [[_ username password] + (re-matches #"([^:]+):([^:]+)" auth-stuff)] + (client/get (str + protocol + rest-of-it) + { + :basic-auth [(java.net.URLDecoder/decode username) + (java.net.URLDecoder/decode password)]}) + (if-let [[_ headerkey headerval] + (re-matches #"([^=]+)=([^=]+)" auth-stuff)] + (client/get (dbg (str + protocol + rest-of-it)) + (dbg { + :headers + { + (keyword (java.net.URLDecoder/decode headerkey)) + (java.net.URLDecoder/decode headerval)}})) + (client/get (str + protocol + rest-of-it) + { + :oauth-token (java.net.URLDecoder/decode auth-stuff) + })))) + (client/get resource)) + (base-slurp resource))) (defn default-spit [loc stuff] diff --git a/test/degasolv/util/core_test.clj b/test/degasolv/util/core_test.clj index 8eb26ab..9c90344 100644 --- a/test/degasolv/util/core_test.clj +++ b/test/degasolv/util/core_test.clj @@ -21,6 +21,44 @@ :Content-Type "text/plain" } }}) + (stub + {:request + { + :method "GET" + :url "/raw-query-string" + :queryParameters { + :in { :equalTo "tact" } + } + } + :response + { + :status 200 + :body "raw query string" + :headers { + :Content-Type "text/plain" + } + }}) + (stub + { + :request + { + :method "GET" + :url "/header-auth" + :headers { + :X-Auth-Token { :equalTo "da7a=" } + } + :queryParameters { + :q { :equalTo "s" } + } + } + :response + { + :status 200 + :body "header auth" + :headers { + :Content-Type "text/plain" + } + }}) (stub {:request { @@ -56,19 +94,11 @@ } } }) - (testing "Basic authentication" + (testing "different authentication techniques" + (is (= "raw query string" + (default-slurp "http://localhost:8080/raw-query-string?in=tact"))) (is (= "bay" (default-slurp "http://localhost:8080/unauthenticated"))) (is (= "boy oh boy" (default-slurp "http://abc:123@localhost:8080/basic-auth"))) (is (= "ay" (default-slurp "http://deadbeef@localhost:8080/bearer-auth"))) - (stop wiremock-server)))) - -(deftest ^:unit-tests assoc-conj-basic - (testing "Add to a blank map" - (is (.equals {:a [1]} - (assoc-conj {} :a 1)))) - (testing "Add to a map with an empty list" - (is (.equals {:a [1]} - (assoc-conj {:a []} :a 1)))) - (testing "Add to a map with an existing list" - (is (.equals {:a [1 2]} - (assoc-conj {:a [1]} :a 2))))) + (is (= "header auth" (default-slurp "http://X-Auth-Token=da7a%3D@localhost:8080/header-auth?q=s")))) + (stop wiremock-server))) From a9595a411653302316f3e63a5f43473641695181 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Fri, 15 Nov 2019 16:41:46 -0700 Subject: [PATCH 17/31] Fix tests surrounding default-slurp --- src/degasolv/util.clj | 64 +++++------ test/degasolv/util/core_test.clj | 184 +++++++++++++++---------------- 2 files changed, 121 insertions(+), 127 deletions(-) diff --git a/src/degasolv/util.clj b/src/degasolv/util.clj index 4e477b5..5c403c5 100644 --- a/src/degasolv/util.clj +++ b/src/degasolv/util.clj @@ -4,13 +4,6 @@ [clojure.pprint :as pprint] [clj-http.client :as client])) - -(defmacro dbg [body] - `(let [x# ~body] - (println "dbg:" '~body "=" x#) - (flush) - x#)) - ; UTF-8 by default :) (defn base-slurp [loc] (let [input (if (= loc "-") @@ -20,34 +13,35 @@ (defn default-slurp [resource] (if (re-matches #"https?://.*" resource) - (if-let [[whole-thing protocol auth-stuff rest-of-it] - (re-matches #"(https?://)([^@]+)@(.+)" resource)] - (:body (if-let [[_ username password] - (re-matches #"([^:]+):([^:]+)" auth-stuff)] - (client/get (str - protocol - rest-of-it) - { - :basic-auth [(java.net.URLDecoder/decode username) - (java.net.URLDecoder/decode password)]}) - (if-let [[_ headerkey headerval] - (re-matches #"([^=]+)=([^=]+)" auth-stuff)] - (client/get (dbg (str - protocol - rest-of-it)) - (dbg { - :headers - { - (keyword (java.net.URLDecoder/decode headerkey)) - (java.net.URLDecoder/decode headerval)}})) - (client/get (str - protocol - rest-of-it) - { - :oauth-token (java.net.URLDecoder/decode auth-stuff) - })))) - (client/get resource)) - (base-slurp resource))) + (:body + (if-let [[whole-thing protocol auth-stuff rest-of-it] + (re-matches #"(https?://)([^@]+)@(.+)" resource)] + (if-let [[_ username password] + (re-matches #"([^:]+):([^:]+)" auth-stuff)] + (client/get (str + protocol + rest-of-it) + { + :basic-auth [(java.net.URLDecoder/decode username) + (java.net.URLDecoder/decode password)]}) + (if-let [[_ headerkey headerval] + (re-matches #"([^=]+)=([^=]+)" auth-stuff)] + (client/get (str + protocol + rest-of-it) + { + :headers + { + (keyword (java.net.URLDecoder/decode headerkey)) + (java.net.URLDecoder/decode headerval)}}) + (client/get (str + protocol + rest-of-it) + { + :oauth-token (java.net.URLDecoder/decode auth-stuff) + }))) + (client/get resource))) + (base-slurp resource))) (defn default-spit [loc stuff] diff --git a/test/degasolv/util/core_test.clj b/test/degasolv/util/core_test.clj index 9c90344..166e9ad 100644 --- a/test/degasolv/util/core_test.clj +++ b/test/degasolv/util/core_test.clj @@ -5,100 +5,100 @@ (deftest ^:integration-tests slurp-auth - (let [wiremock-server (server)] - (start wiremock-server) - (stub - {:request - { - :method "GET" - :url "/unauthenticated" - } - :response - { - :status 200 - :body "bay" - :headers { - :Content-Type "text/plain" - } - }}) - (stub - {:request - { - :method "GET" - :url "/raw-query-string" - :queryParameters { - :in { :equalTo "tact" } - } - } - :response - { - :status 200 - :body "raw query string" - :headers { - :Content-Type "text/plain" - } - }}) - (stub - { - :request - { - :method "GET" - :url "/header-auth" - :headers { - :X-Auth-Token { :equalTo "da7a=" } - } - :queryParameters { - :q { :equalTo "s" } - } - } - :response - { - :status 200 - :body "header auth" - :headers { - :Content-Type "text/plain" - } - }}) - (stub - {:request - { - :method "GET" - :url "/bearer-auth" - :headers { - :Authorization { :equalTo "Bearer deadbeef" } - } - } - :response - { - :status 200 - :body "ay" - :headers { - :Content-Type "text/plain" - } - }}) - (stub - {:request - { - :method "GET" - :url "/basic-auth" - :headers { - :Authorization { :equalTo "Basic YWJjOjEyMw==" } - } - } - :response - { - :status 200 - :body "boy oh boy" - :headers { - :Content-Type "text/plain" - } - } - }) - (testing "different authentication techniques" + (testing "different authentication techniques" + (let [wiremock-server (server)] + (start wiremock-server) + (stub + {:request + { + :method "GET" + :urlPath "/unauthenticated" + } + :response + { + :status 200 + :body "bay" + :headers { + :Content-Type "text/plain" + } + }}) + (stub + {:request + { + :method "GET" + :urlPath "/raw-query-string" + :queryParameters { + :in { :equalTo "tact" } + } + } + :response + { + :status 200 + :body "raw query string" + :headers { + :Content-Type "text/plain" + } + }}) + (stub + { + :request + { + :method "GET" + :urlPath "/header-auth" + :headers { + :X-Auth-Token { :equalTo "da7a=" } + } + :queryParameters { + :q { :equalTo "s" } + } + } + :response + { + :status 200 + :body "header auth" + :headers { + :Content-Type "text/plain" + } + }}) + (stub + {:request + { + :method "GET" + :urlPath "/bearer-auth" + :headers { + :Authorization { :equalTo "Bearer deadbeef" } + } + } + :response + { + :status 200 + :body "ay" + :headers { + :Content-Type "text/plain" + } + }}) + (stub + {:request + { + :method "GET" + :urlPath "/basic-auth" + :headers { + :Authorization { :equalTo "Basic YWJjOjEyMw==" } + } + } + :response + { + :status 200 + :body "boy oh boy" + :headers { + :Content-Type "text/plain" + } + } + }) (is (= "raw query string" (default-slurp "http://localhost:8080/raw-query-string?in=tact"))) (is (= "bay" (default-slurp "http://localhost:8080/unauthenticated"))) (is (= "boy oh boy" (default-slurp "http://abc:123@localhost:8080/basic-auth"))) (is (= "ay" (default-slurp "http://deadbeef@localhost:8080/bearer-auth"))) - (is (= "header auth" (default-slurp "http://X-Auth-Token=da7a%3D@localhost:8080/header-auth?q=s")))) - (stop wiremock-server))) + (is (= "header auth" (default-slurp "http://X-Auth-Token=da7a%3D@localhost:8080/header-auth?q=s"))) + (stop wiremock-server)))) From 840562ce42da8934fffd4b0356e498046df8a886 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Fri, 15 Nov 2019 17:05:00 -0700 Subject: [PATCH 18/31] All the needed documentation exists and is there --- docs/command-reference.rst | 41 +++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/docs/command-reference.rst b/docs/command-reference.rst index 33d6ab7..19fe658 100644 --- a/docs/command-reference.rst +++ b/docs/command-reference.rst @@ -4,7 +4,8 @@ Degasolv Command Reference ========================== This guide describes the Degasolv CLI, what subcommands and options -there are, and what they are for. +there are, and what they are for. It also describes how to specify +options. Some Notes on Versions ---------------------- @@ -73,14 +74,44 @@ A Note on Specifying Files As of version 1.3.0, The whenever an option takes a file in Degasolv, the user can actually specify one of three things: - 1. An ``http://`` or ``https://`` URL. No authentication is - currently supported. + 1. An ``http://`` or ``https://`` URL. Prior to version 2.2.0, no + authentication was supported. As of version 2.2.0, authentication + can be specified in one of three ways: + + 1. **HTTP Basic Authentication**: You can specify a URL-encoded username + and password to use HTTP basic authentication by separating the + username and password via a ``:`` (colon) character and put the + entire thing before the ``@`` character in the URL. For example:: + + https://username:password@example.com/... + + 2. **OAuth2 Token Authentication**: You can specify a URL-encoded + `OAuth2 token`_ by simply specifying one string before the ``@`` in + any given URL, without any separators, like this:: + + https://thisisthetoken@example.com/... + + 3. **Header-Based Authentication**: You can specify a custom HTTP + header, together with its URL encoded value, by separating the + header name from the value of the header with an ``=`` (equals) sign + before the ``@`` in any given URL, like this:: + + https://X-Auth-Token=feefiefofum@example.com/... + + This would yield a HTTP GET request with the following header:: + + X-Auth-Token: feefiefofum + + As of version 2.2.0, query strings as part of the HTTP URL are also + supported. 2. A ``file://`` URL. 3. A filesystem reference. 4. The character ``-``, signifying standard input to the Degasolv process. This is true for options of Degasolv and options for any of its subcommands. +.. _OAuth2 token: https://tools.ietf.org/html/rfc6750 + Explanation of Options ++++++++++++++++++++++ @@ -114,8 +145,8 @@ to it. * The ``:meta`` option is the only option that takes a map or dictionary of values. In this option, keys and values are separated by the equals sign - (``=``) and the list of key/value pairs are also separated by the caret - character, as in ``k=v^k=v^k=v...`` + (``=``) and the list of key/value pairs are also separated by the caret + character, as in ``k=v^k=v^k=v...`` .. note:: The environment variables and their formatting will be listed for the options of all the subcommands in this document; From d0708ef034cfd6bbd28bb6b6a66dd8608fe5cc53 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Fri, 15 Nov 2019 22:05:26 -0700 Subject: [PATCH 19/31] Command reference added security considerations --- docs/command-reference.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/command-reference.rst b/docs/command-reference.rst index 19fe658..4842800 100644 --- a/docs/command-reference.rst +++ b/docs/command-reference.rst @@ -482,6 +482,37 @@ Here is a version of that example that uses JSON instead:: .. _option-pack: .. _option pack: +Security Considerations +*********************** + +Some configuration items in Degasolv, such as URLs that point to config files +and repository indexes, may have passwords or API tokens in-line in the URL. +As an admin, you're going to have to figure out how you want to get sensitive +information of this kind into the configuration for Degasolv to consume. + +There are three main ways to do this for the purposes of security: + + 1. **Environment Variable**: Any option in Degasolv can be specified using + environment variables. See the `environment-variables`_ section. + 2. **Standard Input**: Degasolv configuration can be specified using + standard input. This is probably the most secure and least convenient + way of providing sensitive information to Degasolv. Here is + an example:: + + degasolv -j - << DEGASOLV_CONFIG + + DEGASOLV_CONFIG + + 3. **Configuration File**: This method is somewhat secure as long + as the filesystem is deemed trustworthy and as long as the proper + file permissions are in place so that the credentials can only be viewed + by approved users. + 4. **CLI**: This is not normally secure, but Degasolv leaves the decision + of what is sufficiently secure to the user, and allows sensitive + information to be specified on the command line in the normal way + in an effort to make extra, extra sure the tool is usable even + in a firestorm (read: even in the presence of bizarre use cases). + Option Packs ************ From a2d3913583dff7ba998574790148cf7b5ef8461c Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sat, 16 Nov 2019 14:32:27 -0700 Subject: [PATCH 20/31] Update changelog, prepare for release --- .gitignore | 1 + docs/changelog.rst | 12 ++++++------ src/degasolv/cli.clj | 1 - src/degasolv/util.clj | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index f80083f..9fc34af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /target .cpcache /pkg/ +/__files/ /presentation/target/ /presentation/auto/ /presentation/_minted-*/ diff --git a/docs/changelog.rst b/docs/changelog.rst index 8186b8b..2d83403 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,7 +20,7 @@ Added * Configuration file tower: Degasolv looks for config files in pre-defined locations on the file system if they exist, see - :ref:`Default Configuration Files ` + :ref:`Default Configuration Files `. * Environment variable support: environment variables are consulted first, but are merged into the option map AFTER config file material and BEFORE @@ -28,6 +28,9 @@ Added :ref:`Environment Variables ` and :ref:`How Options are Gathered ` +* Added support for HTTP/HTTPS authentication using basic, oauth2 token + and header methods, see :ref:`A Note on Specifying Files `. + Changed +++++++ @@ -37,11 +40,8 @@ Fixed +++++ * Option packs are expanded at the level upon which they are defined, then the - options are merged together. This seemed like a bug. Though this change is - breaking, we made an exception and released it as part of a minor patch - release, hoping that it would unbreak more than it would break, and knowing - if it broke anybody that they wouldn't be left without a way to fix their - scripts. See :ref:`Option Packs ` for more information. + options are merged together. This seemed like a bug that needed fixing. + See :ref:`Option Packs ` for more information. `2.1.0`_ ------------- diff --git a/src/degasolv/cli.clj b/src/degasolv/cli.clj index 9fa138d..a1d6dde 100644 --- a/src/degasolv/cli.clj +++ b/src/degasolv/cli.clj @@ -38,7 +38,6 @@ (:version %1) (:version %2)))))) - (def ^:private version-comparators diff --git a/src/degasolv/util.clj b/src/degasolv/util.clj index 5c403c5..63f11c0 100644 --- a/src/degasolv/util.clj +++ b/src/degasolv/util.clj @@ -12,8 +12,8 @@ (clojure.core/slurp input :encoding "UTF-8"))) (defn default-slurp [resource] - (if (re-matches #"https?://.*" resource) - (:body + (if (re-matches #"https?://.*" (str resource)) + (:body (if-let [[whole-thing protocol auth-stuff rest-of-it] (re-matches #"(https?://)([^@]+)@(.+)" resource)] (if-let [[_ username password] @@ -41,7 +41,7 @@ :oauth-token (java.net.URLDecoder/decode auth-stuff) }))) (client/get resource))) - (base-slurp resource))) + (base-slurp resource))) (defn default-spit [loc stuff] From cb622703ad2ddd97080b9de53e9a3333bf557cf1 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sat, 16 Nov 2019 15:14:07 -0700 Subject: [PATCH 21/31] Get rid of broken and superfluous script --- test/resources/scripts/test-http-auth | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100755 test/resources/scripts/test-http-auth diff --git a/test/resources/scripts/test-http-auth b/test/resources/scripts/test-http-auth deleted file mode 100755 index 2a6ae0e..0000000 --- a/test/resources/scripts/test-http-auth +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -export POSIXLY_CORRECT=1 -set -x -if [ ! -f './project.clj' -a ! -f './build.boot' ] -then - echo "This script must be run from the root of the project." -fi - -test_home=test/resources/data/test-http-auth - - - -name=$(lein print :name | sed 's|"||g') -version=$(lein print :version | sed 's|"||g') - -java -jar test/resources/data/test-http-auth/wiremock- - -java \ - -Xss16m \ - -jar target/uberjar/${name}-${version}-standalone.jar \ - query-repo \ - --repository "binary-amd64 file://${PWD}/${test_home}/avail /" \ - --package-system apt \ - --query "libc6" - -try_time java \ -lein run -- query-repo --repository http://abc:123@k70rg.mocklab.io/thing/8?fie=ongoodness --query d From bdae67ee8ca5ac5fec7d0a3b4ac4398bb4c1f95d Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sat, 16 Nov 2019 15:36:40 -0700 Subject: [PATCH 22/31] Remove checking for file's existence, as it may or may not be a file --- src/degasolv/cli.clj | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/degasolv/cli.clj b/src/degasolv/cli.clj index a1d6dde..b56615c 100644 --- a/src/degasolv/cli.clj +++ b/src/degasolv/cli.clj @@ -921,19 +921,7 @@ (let [{:keys [options arguments]} (parseplz! subcommand (rest arguments) subcmd-cli) config-files - (get (reduce (fn [[seen result] {^java.io.File new :file read-fn :read-fn - :as unit}] - (let [abs-path (.getAbsolutePath new)] - (if (or (seen abs-path) - (not (.exists new))) - [seen result] - [(conj seen abs-path) - (conj result unit)]))) - [#{} []] - (reduce - into - [] - [(if-let [app-data (System/getenv "AppData")] + (as-> [(if-let [app-data (System/getenv "AppData")] [ {:file (io/file (string/join java.io.File/separator [app-data @@ -966,7 +954,8 @@ :read-fn #(json/parse-string % true)} ] (:config-files env-vars) - (:config-files global-options)])) 1) + (:config-files global-options)] it + (reduce into [] it)) config (get-config config-files) expanded-cfg (expand-option-packs config) From 8e11be1926433434e7a89f7aa77defc70ade7a68 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sat, 16 Nov 2019 17:31:15 -0700 Subject: [PATCH 23/31] Simplify 3rd party licenses --- docs/3rd-party-licenses.rst | 383 +++++------------------------------- 1 file changed, 52 insertions(+), 331 deletions(-) diff --git a/docs/3rd-party-licenses.rst b/docs/3rd-party-licenses.rst index ea5bcfd..7501d89 100644 --- a/docs/3rd-party-licenses.rst +++ b/docs/3rd-party-licenses.rst @@ -6,334 +6,55 @@ Degasolv is built on the shoulders of giants. Here are the licenses for the third party software that comes with Degasolv. -Clojure and Clojure Libraries ------------------------------ - -All of Clojure, the language in which Degasolv was written, and the clojure -libraries it uses all use the same license: The Eclipse Public License, -version 1.0. It is presented below along with the names of the libraries used -in Degasolv. - -Clojure -+++++++ - -+------------------------------+--------------------------------------+ -| Software | `Clojure Programming Language`_ | -+------------------------------+--------------------------------------+ -| Author(s) | Rich Hickey and Contributors | -+------------------------------+--------------------------------------+ -| Version of software | 1.10.1 | -+------------------------------+--------------------------------------+ -| Degasolv version introduced | 1.0.2 | -+------------------------------+--------------------------------------+ - -.. _Clojure Programming Language: https://github.com/clojure/clojure/releases/tag/clojure-1.10.1 - -serovers -++++++++ - -+------------------------------+--------------------------------------+ -| Software | `serovers`_ | -+------------------------------+--------------------------------------+ -| Author(s) | Daniel Jay Haskin | -+------------------------------+--------------------------------------+ -| Version of software | 1.6.2 | -+------------------------------+--------------------------------------+ -| Degasolv version introduced | 1.8.0 | -+------------------------------+--------------------------------------+ - -.. _Serovers: https://gitlab.com/djhaskin987/serovers/-/tags/1.6.2 - - -tools.cli -+++++++++ - -+------------------------------+--------------------------------------+ -| Software | `tools.cli`_ | -+------------------------------+--------------------------------------+ -| Author(s) | Rich Hickey and contributors | -+------------------------------+--------------------------------------+ -| Version of software | 0.3.5 | -+------------------------------+--------------------------------------+ -| Degasolv version introduced | 1.0.2 | -+------------------------------+--------------------------------------+ - -.. _tools.cli: https://github.com/clojure/tools.cli/releases/tag/tools.cli-0.3.5 - -tagged -++++++ - -+------------------------------+--------------------------------------+ -| Software | `tagged`_ | -+------------------------------+--------------------------------------+ -| Author(s) | Stephen E. Miner | -+------------------------------+--------------------------------------+ -| Version of software | 0.5.0 | -+------------------------------+--------------------------------------+ -| Degasolv version introduced | 1.0.2 | -+------------------------------+--------------------------------------+ - -.. _tagged: https://github.com/miner/tagged - -data.json -+++++++++ - -+------------------------------+--------------------------------------+ -| Software | `data.json`_ | -+------------------------------+--------------------------------------+ -| Author(s) | Stuart Sierra | -+------------------------------+--------------------------------------+ -| Version of software | 0.2.6 | -+------------------------------+--------------------------------------+ -| Degasolv version introduced | 1.10.0 | -+------------------------------+--------------------------------------+ - -.. _data.json: https://github.com/clojure/data.json/releases/tag/data.json-0.2.6 - - -core.match -++++++++++ - -+------------------------------+--------------------------------------+ -| Software | `core.match`_ | -+------------------------------+--------------------------------------+ -| Author(s) | David Nolen, Ambrose Bonnaire- | -| | Sergeant, Rich Hickey & contributors | -+------------------------------+--------------------------------------+ -| Version of software | 0.3.0-alpha5 | -+------------------------------+--------------------------------------+ -| Degasolv version introduced | 1.0.2 | -+------------------------------+--------------------------------------+ - -.. _core.match: https://github.com/clojure/core.match/releases/tag/core.match-0.3.0-alpha5 - -Eclipse Public License -++++++++++++++++++++++ - - Eclipse Public License - v 1.0 - - THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE - PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF - THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. - - 1. DEFINITIONS - - "Contribution" means: - - a) in the case of the initial Contributor, the initial code and - documentation distributed under this Agreement, and - - b) in the case of each subsequent Contributor: - - i) changes to the Program, and - - ii) additions to the Program; - - where such changes and/or additions to the Program originate from and - are distributed by that particular Contributor. A Contribution - 'originates' from a Contributor if it was added to the Program by such - Contributor itself or anyone acting on such Contributor's behalf. - Contributions do not include additions to the Program which: (i) are - separate modules of software distributed in conjunction with the - Program under their own license agreement, and (ii) are not derivative - works of the Program. - - "Contributor" means any person or entity that distributes the Program. - - "Licensed Patents" mean patent claims licensable by a Contributor which - are necessarily infringed by the use or sale of its Contribution alone - or when combined with the Program. - - "Program" means the Contributions distributed in accordance with this - Agreement. - - "Recipient" means anyone who receives the Program under this Agreement, - including all Contributors. - - 2. GRANT OF RIGHTS - - a) Subject to the terms of this Agreement, each Contributor hereby - grants Recipient a non-exclusive, worldwide, royalty-free copyright - license to reproduce, prepare derivative works of, publicly display, - publicly perform, distribute and sublicense the Contribution of such - Contributor, if any, and such derivative works, in source code and - object code form. - - b) Subject to the terms of this Agreement, each Contributor hereby - grants Recipient a non-exclusive, worldwide, royalty-free patent - license under Licensed Patents to make, use, sell, offer to sell, - import and otherwise transfer the Contribution of such Contributor, if - any, in source code and object code form. This patent license shall - apply to the combination of the Contribution and the Program if, at the - time the Contribution is added by the Contributor, such addition of the - Contribution causes such combination to be covered by the Licensed - Patents. The patent license shall not apply to any other combinations - which include the Contribution. No hardware per se is licensed - hereunder. - - c) Recipient understands that although each Contributor grants the - licenses to its Contributions set forth herein, no assurances are - provided by any Contributor that the Program does not infringe the - patent or other intellectual property rights of any other entity. Each - Contributor disclaims any liability to Recipient for claims brought by - any other entity based on infringement of intellectual property rights - or otherwise. As a condition to exercising the rights and licenses - granted hereunder, each Recipient hereby assumes sole responsibility to - secure any other intellectual property rights needed, if any. For - example, if a third party patent license is required to allow Recipient - to distribute the Program, it is Recipient's responsibility to acquire - that license before distributing the Program. - - d) Each Contributor represents that to its knowledge it has sufficient - copyright rights in its Contribution, if any, to grant the copyright - license set forth in this Agreement. - - 3. REQUIREMENTS - - A Contributor may choose to distribute the Program in object code form - under its own license agreement, provided that: - - a) it complies with the terms and conditions of this Agreement; and - - b) its license agreement: - - i) effectively disclaims on behalf of all Contributors all warranties - and conditions, express and implied, including warranties or conditions - of title and non-infringement, and implied warranties or conditions of - merchantability and fitness for a particular purpose; - - ii) effectively excludes on behalf of all Contributors all liability - for damages, including direct, indirect, special, incidental and - consequential damages, such as lost profits; - - iii) states that any provisions which differ from this Agreement are - offered by that Contributor alone and not by any other party; and - - iv) states that source code for the Program is available from such - Contributor, and informs licensees how to obtain it in a reasonable - manner on or through a medium customarily used for software exchange. - - When the Program is made available in source code form: - - a) it must be made available under this Agreement; and - - b) a copy of this Agreement must be included with each copy of the - Program. - - Contributors may not remove or alter any copyright notices contained - within the Program. - - Each Contributor must identify itself as the originator of its - Contribution, if any, in a manner that reasonably allows subsequent - Recipients to identify the originator of the Contribution. - - 4. COMMERCIAL DISTRIBUTION - - Commercial distributors of software may accept certain responsibilities - with respect to end users, business partners and the like. While this - license is intended to facilitate the commercial use of the Program, - the Contributor who includes the Program in a commercial product - offering should do so in a manner which does not create potential - liability for other Contributors. Therefore, if a Contributor includes - the Program in a commercial product offering, such Contributor - ("Commercial Contributor") hereby agrees to defend and indemnify every - other Contributor ("Indemnified Contributor") against any losses, - damages and costs (collectively "Losses") arising from claims, lawsuits - and other legal actions brought by a third party against the - Indemnified Contributor to the extent caused by the acts or omissions - of such Commercial Contributor in connection with its distribution of - the Program in a commercial product offering. The obligations in this - section do not apply to any claims or Losses relating to any actual or - alleged intellectual property infringement. In order to qualify, an - Indemnified Contributor must: a) promptly notify the Commercial - Contributor in writing of such claim, and b) allow the Commercial - Contributor to control, and cooperate with the Commercial Contributor - in, the defense and any related settlement negotiations. The - Indemnified Contributor may participate in any such claim at its own - expense. - - For example, a Contributor might include the Program in a commercial - product offering, Product X. That Contributor is then a Commercial - Contributor. If that Commercial Contributor then makes performance - claims, or offers warranties related to Product X, those performance - claims and warranties are such Commercial Contributor's responsibility - alone. Under this section, the Commercial Contributor would have to - defend claims against the other Contributors related to those - performance claims and warranties, and if a court requires any other - Contributor to pay any damages as a result, the Commercial Contributor - must pay those damages. - - 5. NO WARRANTY - - EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS - PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY - WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR - FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible - for determining the appropriateness of using and distributing the - Program and assumes all risks associated with its exercise of rights - under this Agreement , including but not limited to the risks and costs - of program errors, compliance with applicable laws, damage to or loss - of data, programs or equipment, and unavailability or interruption of - operations. - - 6. DISCLAIMER OF LIABILITY - - EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR - ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING - WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR - DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED - HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - - 7. GENERAL - - If any provision of this Agreement is invalid or unenforceable under - applicable law, it shall not affect the validity or enforceability of - the remainder of the terms of this Agreement, and without further - action by the parties hereto, such provision shall be reformed to the - minimum extent necessary to make such provision valid and enforceable. - - If Recipient institutes patent litigation against any entity (including - a cross-claim or counterclaim in a lawsuit) alleging that the Program - itself (excluding combinations of the Program with other software or - hardware) infringes such Recipient's patent(s), then such Recipient's - rights granted under Section 2(b) shall terminate as of the date such - litigation is filed. - - All Recipient's rights under this Agreement shall terminate if it fails - to comply with any of the material terms or conditions of this - Agreement and does not cure such failure in a reasonable period of time - after becoming aware of such noncompliance. If all Recipient's rights - under this Agreement terminate, Recipient agrees to cease use and - distribution of the Program as soon as reasonably practicable. However, - Recipient's obligations under this Agreement and any licenses granted - by Recipient relating to the Program shall continue and survive. - - Everyone is permitted to copy and distribute copies of this Agreement, - but in order to avoid inconsistency the Agreement is copyrighted and - may only be modified in the following manner. The Agreement Steward - reserves the right to publish new versions (including revisions) of - this Agreement from time to time. No one other than the Agreement - Steward has the right to modify this Agreement. The Eclipse Foundation - is the initial Agreement Steward. The Eclipse Foundation may assign the - responsibility to serve as the Agreement Steward to a suitable separate - entity. Each new version of the Agreement will be given a - distinguishing version number. The Program (including Contributions) - may always be distributed subject to the version of the Agreement under - which it was received. In addition, after a new version of the - Agreement is published, Contributor may elect to distribute the Program - (including its Contributions) under the new version. Except as - expressly stated in Sections 2(a) and 2(b) above, Recipient receives no - rights or licenses to the intellectual property of any Contributor - under this Agreement, whether expressly, by implication, estoppel or - otherwise. All rights in the Program not expressly granted under this - Agreement are reserved. - - This Agreement is governed by the laws of the State of New York and the - intellectual property laws of the United States of America. No party to - this Agreement will bring a legal action under this Agreement more than - one year after the cause of action arose. Each party waives its rights - to a jury trial in any resulting litigation. +``` +org.skyscreamer/jsonassert - 1.2.3 - The Apache Software License, Version 2.0 +nrepl - 0.6.0 - Eclipse Public License +commons-io - 2.6 - Apache License, Version 2.0 +clojure-complete - 0.2.5 - Eclipse Public License +org.apache.httpcomponents/httpmime - 4.5.8 - Apache License, Version 2.0 +potemkin - 0.4.5 - MIT License +com.fasterxml.jackson.dataformat/jackson-dataformat-smile - 2.9.9 - The Apache Software License, Version 2.0 +clj-wiremock - 0.3.0 - Eclipse Public License (present in tests only; not +released) +cheshire - 5.9.0 - The MIT License +pjstadig/humane-test-output - 0.9.0 - Eclipse Public License +clj-http - 3.10.0 - The MIT License +org.apache.httpcomponents/httpclient - 4.5.8 - Apache License, Version 2.0 +org.clojure/tools.cli - 0.3.5 - Eclipse Public License 1.0 +com.fasterxml.jackson.core/jackson-databind - 2.4.2 - The Apache Software License, Version 2.0 +net.sf.jopt-simple/jopt-simple - 4.7 - The MIT License +org.json/json - 20090211 - provided without support or warranty +org.mortbay.jetty/jetty - 6.1.26 - Apache Software License - Version 2.0 +com.fasterxml.jackson.dataformat/jackson-dataformat-cbor - 2.9.9 - The Apache Software License, Version 2.0 +com.jayway.jsonpath/json-path - 0.8.1 - Apache 2.0 +org.flatland/ordered - 1.5.7 - Eclipse Public License - v 1.0 +net.minidev/json-smart - 1.1.1 - The Apache Software License, Version 2.0 +com.google.guava/guava - 18.0 - The Apache Software License, Version 2.0 +tigris - 0.1.1 - Eclipse Public License +clj-tuple - 0.2.2 - MIT License +org.mortbay.jetty/jetty-util - 6.1.26 - Apache Software License - Version 2.0 +commons-lang - 2.6 - The Apache Software License, Version 2.0 +org.apache.httpcomponents/httpcore - 4.4.11 - Apache License, Version 2.0 +slingshot - 0.12.2 - Eclipse Public License 1.0 +org.apache.httpcomponents/httpclient-cache - 4.5.8 - Apache License, Version 2.0 +org.apache.httpcomponents/httpasyncclient - 4.1.4 - Apache License, Version 2.0 +org.mortbay.jetty/servlet-api - 2.5-20081211 - Apache License Version 2.0 +org.flatland/useful - 0.11.6 - Eclipse Public License - v 1.0 +com.fasterxml.jackson.core/jackson-core - 2.9.9 - The Apache Software License, Version 2.0 +org.clojure/tools.reader - 0.7.2 - Eclipse Public License 1.0 +org.clojure/core.match - 0.3.0-alpha5 - Eclipse Public License 1.0 (present in tests only; not released) +org.clojure/clojure - 1.10.1 - Eclipse Public License 1.0 +org.clojure/spec.alpha - 0.2.176 - Eclipse Public License 1.0 +serovers - 1.6.2 - Eclipse Public License +riddley - 0.1.12 - MIT License +commons-logging - 1.2 - The Apache Software License, Version 2.0 +com.fasterxml.jackson.core/jackson-annotations - 2.4.2 - The Apache Software License, Version 2.0 +org.apache.httpcomponents/httpcore-nio - 4.4.10 - Apache License, Version 2.0 +org.slf4j/slf4j-api - 1.7.6 - MIT License +org.clojure/tools.macro - 0.1.1 - Eclipse Public License 1.0 +com.velisco/tagged - 0.5.0 - Eclipse Public License +org.clojure/core.specs.alpha - 0.2.44 - Eclipse Public License 1.0 +com.github.tomakehurst/wiremock - 1.54 - The Apache Software License, Version 2.0 (present in tests only; not released) +commons-codec - 1.12 - Apache License, Version 2.0 +xmlunit - 1.5 - BSD License +``` From d86f35725921178f001c457f1512dbd06874781c Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sat, 16 Nov 2019 17:33:48 -0700 Subject: [PATCH 24/31] Bump version number --- docs/changelog.rst | 15 ++++++++++++++- project.clj | 4 ++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2d83403..a044c40 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,18 @@ and this project adheres to `Semantic Versioning`_. Added +++++ +Changed ++++++++ + +Fixed ++++++ + +`2.2.0`_ +-------- + +Added ++++++ + * Configuration file tower: Degasolv looks for config files in pre-defined locations on the file system if they exist, see :ref:`Default Configuration Files `. @@ -382,7 +394,8 @@ Added * This isn't the first release, but for the purposes of these docs, it is :D -.. _Unreleased: https://github.com/djhaskin987/degasolv/compare/2.1.0...HEAD +.. _Unreleased: https://github.com/djhaskin987/degasolv/compare/2.2.0...HEAD +.. _2.2.0: https://github.com/djhaskin987/degasolv/compare/2.1.0...2.2.0 .. _2.1.0: https://github.com/djhaskin987/degasolv/compare/2.0.0...2.1.0 .. _2.0.0: https://github.com/djhaskin987/degasolv/compare/1.12.1...2.0.0 .. _1.12.1: https://github.com/djhaskin987/degasolv/compare/1.12.0...1.12.1 diff --git a/project.clj b/project.clj index 94fd847..40c6ab0 100644 --- a/project.clj +++ b/project.clj @@ -1,5 +1,5 @@ -(defproject degasolv/degasolv "2.2.0-SNAPSHOT" - :description "Dependency tracker with an eye toward building and shipping software." +(defproject degasolv/degasolv "2.2.0" + :description "Democratize dependency management." :url "http://github.com/djhaskin987/degasolv" :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 73b808cc5903ea4abad70327b1007da17961eacf Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sat, 16 Nov 2019 17:38:39 -0700 Subject: [PATCH 25/31] Update 3rd party licenses --- docs/3rd-party-licenses.rst | 84 ++++++++++++++----------------------- 1 file changed, 31 insertions(+), 53 deletions(-) diff --git a/docs/3rd-party-licenses.rst b/docs/3rd-party-licenses.rst index 7501d89..8a74ec7 100644 --- a/docs/3rd-party-licenses.rst +++ b/docs/3rd-party-licenses.rst @@ -4,57 +4,35 @@ ================== Degasolv is built on the shoulders of giants. Here are the licenses for the -third party software that comes with Degasolv. +third party software that comes with Degasolv:: -``` -org.skyscreamer/jsonassert - 1.2.3 - The Apache Software License, Version 2.0 -nrepl - 0.6.0 - Eclipse Public License -commons-io - 2.6 - Apache License, Version 2.0 -clojure-complete - 0.2.5 - Eclipse Public License -org.apache.httpcomponents/httpmime - 4.5.8 - Apache License, Version 2.0 -potemkin - 0.4.5 - MIT License -com.fasterxml.jackson.dataformat/jackson-dataformat-smile - 2.9.9 - The Apache Software License, Version 2.0 -clj-wiremock - 0.3.0 - Eclipse Public License (present in tests only; not -released) -cheshire - 5.9.0 - The MIT License -pjstadig/humane-test-output - 0.9.0 - Eclipse Public License -clj-http - 3.10.0 - The MIT License -org.apache.httpcomponents/httpclient - 4.5.8 - Apache License, Version 2.0 -org.clojure/tools.cli - 0.3.5 - Eclipse Public License 1.0 -com.fasterxml.jackson.core/jackson-databind - 2.4.2 - The Apache Software License, Version 2.0 -net.sf.jopt-simple/jopt-simple - 4.7 - The MIT License -org.json/json - 20090211 - provided without support or warranty -org.mortbay.jetty/jetty - 6.1.26 - Apache Software License - Version 2.0 -com.fasterxml.jackson.dataformat/jackson-dataformat-cbor - 2.9.9 - The Apache Software License, Version 2.0 -com.jayway.jsonpath/json-path - 0.8.1 - Apache 2.0 -org.flatland/ordered - 1.5.7 - Eclipse Public License - v 1.0 -net.minidev/json-smart - 1.1.1 - The Apache Software License, Version 2.0 -com.google.guava/guava - 18.0 - The Apache Software License, Version 2.0 -tigris - 0.1.1 - Eclipse Public License -clj-tuple - 0.2.2 - MIT License -org.mortbay.jetty/jetty-util - 6.1.26 - Apache Software License - Version 2.0 -commons-lang - 2.6 - The Apache Software License, Version 2.0 -org.apache.httpcomponents/httpcore - 4.4.11 - Apache License, Version 2.0 -slingshot - 0.12.2 - Eclipse Public License 1.0 -org.apache.httpcomponents/httpclient-cache - 4.5.8 - Apache License, Version 2.0 -org.apache.httpcomponents/httpasyncclient - 4.1.4 - Apache License, Version 2.0 -org.mortbay.jetty/servlet-api - 2.5-20081211 - Apache License Version 2.0 -org.flatland/useful - 0.11.6 - Eclipse Public License - v 1.0 -com.fasterxml.jackson.core/jackson-core - 2.9.9 - The Apache Software License, Version 2.0 -org.clojure/tools.reader - 0.7.2 - Eclipse Public License 1.0 -org.clojure/core.match - 0.3.0-alpha5 - Eclipse Public License 1.0 (present in tests only; not released) -org.clojure/clojure - 1.10.1 - Eclipse Public License 1.0 -org.clojure/spec.alpha - 0.2.176 - Eclipse Public License 1.0 -serovers - 1.6.2 - Eclipse Public License -riddley - 0.1.12 - MIT License -commons-logging - 1.2 - The Apache Software License, Version 2.0 -com.fasterxml.jackson.core/jackson-annotations - 2.4.2 - The Apache Software License, Version 2.0 -org.apache.httpcomponents/httpcore-nio - 4.4.10 - Apache License, Version 2.0 -org.slf4j/slf4j-api - 1.7.6 - MIT License -org.clojure/tools.macro - 0.1.1 - Eclipse Public License 1.0 -com.velisco/tagged - 0.5.0 - Eclipse Public License -org.clojure/core.specs.alpha - 0.2.44 - Eclipse Public License 1.0 -com.github.tomakehurst/wiremock - 1.54 - The Apache Software License, Version 2.0 (present in tests only; not released) -commons-codec - 1.12 - Apache License, Version 2.0 -xmlunit - 1.5 - BSD License -``` + commons-io - 2.6 - Apache License, Version 2.0 + clojure-complete - 0.2.5 - Eclipse Public License + org.apache.httpcomponents/httpmime - 4.5.8 - Apache License, Version 2.0 + potemkin - 0.4.5 - MIT License + com.fasterxml.jackson.dataformat/jackson-dataformat-smile - 2.9.9 - The Apache Software License, Version 2.0 + cheshire - 5.9.0 - The MIT License + clj-http - 3.10.0 - The MIT License + org.apache.httpcomponents/httpclient - 4.5.8 - Apache License, Version 2.0 + org.clojure/tools.cli - 0.3.5 - Eclipse Public License 1.0 + com.fasterxml.jackson.dataformat/jackson-dataformat-cbor - 2.9.9 - The Apache Software License, Version 2.0 + org.flatland/ordered - 1.5.7 - Eclipse Public License - v 1.0 + tigris - 0.1.1 - Eclipse Public License + clj-tuple - 0.2.2 - MIT License + org.apache.httpcomponents/httpcore - 4.4.11 - Apache License, Version 2.0 + slingshot - 0.12.2 - Eclipse Public License 1.0 + org.apache.httpcomponents/httpclient-cache - 4.5.8 - Apache License, Version 2.0 + org.apache.httpcomponents/httpasyncclient - 4.1.4 - Apache License, Version 2.0 + org.flatland/useful - 0.11.6 - Eclipse Public License - v 1.0 + com.fasterxml.jackson.core/jackson-core - 2.9.9 - The Apache Software License, Version 2.0 + org.clojure/tools.reader - 0.7.2 - Eclipse Public License 1.0 + org.clojure/clojure - 1.10.1 - Eclipse Public License 1.0 + org.clojure/spec.alpha - 0.2.176 - Eclipse Public License 1.0 + serovers - 1.6.2 - Eclipse Public License + riddley - 0.1.12 - MIT License + commons-logging - 1.2 - The Apache Software License, Version 2.0 + org.apache.httpcomponents/httpcore-nio - 4.4.10 - Apache License, Version 2.0 + org.clojure/tools.macro - 0.1.1 - Eclipse Public License 1.0 + com.velisco/tagged - 0.5.0 - Eclipse Public License + org.clojure/core.specs.alpha - 0.2.44 - Eclipse Public License 1.0 + commons-codec - 1.12 - Apache License, Version 2.0 From b6683e50701a6548bf837a6524144c1074493752 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sun, 17 Nov 2019 10:40:12 -0700 Subject: [PATCH 26/31] Prepared roadmap for the next release --- docs/roadmap.rst | 226 ++++++----------------------------------------- 1 file changed, 27 insertions(+), 199 deletions(-) diff --git a/docs/roadmap.rst b/docs/roadmap.rst index 7440700..88a93de 100644 --- a/docs/roadmap.rst +++ b/docs/roadmap.rst @@ -26,204 +26,32 @@ Future Releases 2.3.0 ----- -Package Installation Manager -++++++++++++++++++++++++++++ - -This is what has been missing from Degasolv for a long time: the notion of an -installation. We've got dependency management down; now we need to get into -installation management. - -In keeping with Degasolv's original vision, we will *not* make specific what -is actually being installed or maintained. This way, Degasolv can be kept -as a neutral, generic tool that can apply in many situations. Firefighters -need tools that can apply in many situations; similarly, ops and DevOps -professionals, for whom we build this tool, need to a dependency management -tool that can get them out of dependency hell no matter what their situation. - -Therefore, we focus on two explicit dimensions of installation management, the -"hard parts", and leave the implementation details like unpacking archives and -writing to file or key-value stores to the DevOps professional. - -These are: -1. Resource management -2. Package installation database - -An Installation Database -++++++++++++++++++++++++ - -A new database that represents a particular package "installation". This could -be used for a file installation, a kubernetes/windows/whatever installation. - -package installation will house: -- Site-wide configuration -- Packages that have been installed and the relationships between them - -This database will allow us to track and manage: -- package installations -- package removals -- package verification - -- [ ] A new global option, ``--installation``, that would take a JDBC - connection string. If the option ``--ignore-installation`` were also - present any functionality would be turned off, including ignoring the - installation for configuration items. An optional username and password - for the connection, ``--installation-username``, and ``--installation-password``, - would be allowed. Last wins. - -- [ ] If the ``--installation`` option is present, the ``resolve-locations`` - subcommand would take the installation into account in its resolution as - present packages. - -- [ ] If the ``--enable-installation-only`` is present, the ``resolve-locations`` - subcommand would take the installation into account as its sole index. - Any subsequent options given for other indices will be recorded, but - ignored. These will be turned "back on" if the option - ``--disable-installation-only`` is encountered. - -- [ ] If the ``--enable-from-installation`` is present, the ``resolve-locations`` - subcommand would take the installation into account as one of its - indexes in "the usual order", as if it were another specified index - on the commandline. Its order will be reset if it is encountered again. - It can be turned off with ``--disable-from-installation``. Last wins. - -- [ ] A new subcommand, ``init-installation``, would require that - ``--installation`` were given, optional username and password for the - connection. It would record present configuration (that is, configuration as - observed by init-installation), and set up the necessary tables for future - use. - -- [ ] A new subcommand, ``resolve-dependents``, that takes an installation, - package and version (presumably installed) and lists all packages and - versions that depend on that version of that package. - - - [ ] A new function, ``resolve-dependents``, that takes a package and - version and query function and uses the query function to determine - what other packages depend on the current package, recursively. - -- [ ] A new subcommand, ``install-package``, that would take a plain list or - json blob of packages and installs them provided their resources were - mutually exclusive with each other and all installed packages (see below) and - provided their dependencies were met. this can easily be done by calling - ``resolve-dependencies`` with a nullary query function and big list of - packages to install. Dependencies will not be checked if ``--force`` is - present. - -- [ ] A new subcommand, ``remove-package``, that would take a plain list or - json blob of packages and remove them provided all dependencies were met. - Package dependencies will not be checked if ``--force`` is present. - -- [ ] A new subcommand, ``list-packages``, that would yield to standard out a - plain/json list of packages from an installation. It would take an optional - parameter that lists out a particular package at a particular version, or all - packages matching only a package name. - -A New Notion of Resources -+++++++++++++++++++++++++ - -As part of this new initiative, packages will be given a new attribute called -"resources". This attribute will house a map from resource type names to a map -of resource states. This map will map a resources to their states upon -installation. Both resources and resource states are simply strings. Meta -information under the ``meta`` key will also be stored. - -For example, resources could be files and states could be sha256 hashes, -with any metadata about the files, like whether they are config or not:: - - { - "files": { - "/var/lib/foo": { - "state": "600dc0d36077a10ada600dd3a10fda7a600dc0d36077a10ada600dd3a10fda7a", - "meta": { - "tags": ["config"] - } - } - } - } - -With this new notion of resources, resource management can be correctly -implemented upon package installation, removal, creation, and verification. - -- [ ] A new option needs to be added to ``generate-card`` to specify resources. - This would be an executable that would take a package name, version and - location and would return resources in JSON over standard output. - -- [ ] A new subcommand, ``verify-package``, needs to be created that verifies - listed package resources against listed resources. This would take an - executable as an option that took the name and type of a resource and yielded - a string on standard output that would be the state. A diff of the supposed - state and the actual state would be printed. If no arguments are given, it - lists verification information for all such packages, plain/json. - -- [ ] A feature of ``install-package``: A package can only be installed if no - package, currently being installed or previously installed, installs the same - resources. - - -2.2.0 ------ - -- [ ] Environment-variable specification of options: the ability to specify - options using environment variables. A string will be returned as the value - of the option, unless a caret (``^``) character is present in the string, in - which case the string will be split into a list of strings using the caret - characters as boundaries. Arguments that take a boolean option will atttempt - to parse the string, expecting either the values ``true`` or ``false`` as the - value of the string. It is an error for boolean arguments (e.g., - ``DEGASOLV_ALTERNATIVES`` or ``DEGASOLV_ERROR_FORMAT``) to have any other - string. - -- [ ] Authenticated HTTP and HTTPS connections: we will provide a way by which - HTTP and HTTPS connections are authenticated, I think using a .netrc-like - mapping between hosts and username/password pairs. Documentation should be - written around specifying username and password securely according to needs, - whether by standard input using the conventional config file mechanisms, by - environment variable, or by command line. Each has security implications that - the user needs to be aware of. - -- [ ] Documentation should be written around specifying username and password - securely according to needs, whether by standard input using the conventional - config file mechanisms, by environment variable, or by command line. Each has - security implications that the user needs to be aware of. - - -2.1.0 ------ - -- [ ] **Minimum Version Selection**: Implement minimum version selection as an - option pack. Supporting features: - - [x] Version suggestion: when trying different candidates, if a matching - id is in the return of a candidate, put the suggestions in the list to - loop through. - - - CANCELLED: Skip unlikely candidates: Keep a set of problem ids different - candidates are tried. If the current alternative's ID is not in that set, - and if the dependencies are the same as the previous candidate, then skip - the candidate; do not try it. - - This was cancelled because it was problematic. - - - [ ] Minimum version preference: In generate repo index, add option to - sort packages the other way. - - - [ ] Proper documentation surrounding order of encounter, that for example - for subproc degasolv will honor the order of packages found in the repo - index and that this enables things like MVS. - -- CANCELLED: **Git package system**: Implement git package system that knows - how to read bitbucket, bitbucket server, github, github server, gitlab, - gitlab server, and raw git repos. This was cancelled because it felt "out of - scope"; i didn't want to build ever-changing vendor-specific API details into - a tool I hope will be a more generic tool that outlasts these APIs. - I think git integrations is the subject for a good but different tool. - -- [ ] **Inclusive/Absent corner case**: In the case that ``conflict-strat`` is - inclusive, a case may arise where an absence is required, but then another - requirement asks for the same package to be present at a conflicting version. - basically, what do requirements of absence mean within and inclusive context. - because right now when absence requirements are given, they hold force even - in inclusive contexts. This seems silly. Requirements of absence should just - be ignored when ``conflict-strat`` is ``inclusive``. - -- [ ] Fix #17, it is awful. +Firefighters need tools that can apply in many situations; similarly, ops and +DevOps professionals, for whom we build this tool, need to a dependency +management tool that can get them out of dependency hell no matter what their +situation. +- [ ] Shortened versions of all subcommands, including documentation updates. +- [ ] The ability to slurp from JDBC URLs for indexes + - [ ] An extension will be made to ensure that username and password + can be specified along with a URL. Not all drivers support this + and it is an important use case. + - [ ] Generate-repo-index to support JDBC URLs + - [ ] query-repo and resolve-locations to support JDBC URLs + - [ ] If the database is empty or doesn't exist, it will be created on + generate repo index or on index-add +- [ ] New subcommands: index-add, index-rm, to take away from and add to + as in an installation/removal context +- [ ] New subcommand: resolve-dependents to find all dependents in an *index* +- [ ] USER GUIDES + - [ ] How to use repositories as generic installation trackers + - [ ] How to track dependencies between kubernetes services + - [ ] How to track dependencies between cross-language builds and use this + for that + - [ ] Documentation and/or code on the topic of supporting the use case + of different architectures of the same package using prioritized + indexes of + - [ ] How to use degasolv to manage a project installation for development + purposes + packages named the same with different contents. From 183fe84213dddba1982456ba295e505cf96495c6 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sun, 17 Nov 2019 13:58:21 -0700 Subject: [PATCH 27/31] Clarify roadmap --- docs/roadmap.rst | 55 ++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/docs/roadmap.rst b/docs/roadmap.rst index 88a93de..35a581c 100644 --- a/docs/roadmap.rst +++ b/docs/roadmap.rst @@ -9,16 +9,16 @@ the Degasolv developers. Future Releases --------------- -- [ ] Tutorial-like help screens designed to keep people from needing to switch +- Tutorial-like help screens designed to keep people from needing to switch from docs to cli and back. -- [ ] Shortened versions of all subcommands, including documentation updates. +- Shortened versions of all subcommands, including documentation updates. -- [ ] Documentation and/or code on the topic of supporting the use case of +- Documentation and/or code on the topic of supporting the use case of different architectures of the same package using prioritized indexes of packages named the same with different contents. -- [ ] **Compile with GraalVM's ``native-image``**: Compile degasolv to machine +- **Compile with GraalVM's ``native-image``**: Compile degasolv to machine code with GraalVM's ``native-image`` to decrease start-up times. This will likely coincide with upgrading to Clojure 1.11 because native-image doesn't work with Clojure 1.10.1 . @@ -31,27 +31,28 @@ DevOps professionals, for whom we build this tool, need to a dependency management tool that can get them out of dependency hell no matter what their situation. -- [ ] Shortened versions of all subcommands, including documentation updates. -- [ ] The ability to slurp from JDBC URLs for indexes - - [ ] An extension will be made to ensure that username and password - can be specified along with a URL. Not all drivers support this - and it is an important use case. - - [ ] Generate-repo-index to support JDBC URLs - - [ ] query-repo and resolve-locations to support JDBC URLs - - [ ] If the database is empty or doesn't exist, it will be created on - generate repo index or on index-add -- [ ] New subcommands: index-add, index-rm, to take away from and add to - as in an installation/removal context -- [ ] New subcommand: resolve-dependents to find all dependents in an *index* -- [ ] USER GUIDES - - [ ] How to use repositories as generic installation trackers - - [ ] How to track dependencies between kubernetes services - - [ ] How to track dependencies between cross-language builds and use this - for that - - [ ] Documentation and/or code on the topic of supporting the use case - of different architectures of the same package using prioritized - indexes of - - [ ] How to use degasolv to manage a project installation for development - purposes - packages named the same with different contents. +- Shortened versions of all subcommands, including documentation updates. +- The ability to slurp from JDBC URLs for indexes + - An extension will be made to ensure that username and password + can be specified along with a URL. Not all drivers support this + and it is an important use case. + - Generate-repo-index to support JDBC URLs + - query-repo and resolve-locations to support JDBC URLs + - If the database is empty or doesn't exist, it will be created on + generate repo index or on index-add + +- New subcommands: index-add, index-rm, to take away from and add to + as in an installation/removal context +- New subcommand: resolve-dependents to find all dependents in an *index* +- USER GUIDES +- How to use repositories as generic installation trackers +- How to track dependencies between kubernetes services +- How to track dependencies between cross-language builds and use this for + that + + - Documentation and/or code on the topic of supporting the use case + of different architectures of the same package using prioritized + indexes of packages named the same with different contents. + - How to use degasolv to manage a project installation for + development purposes From 0246198b61f2f7ea431e2c082139c058781bd876 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sun, 17 Nov 2019 13:59:28 -0700 Subject: [PATCH 28/31] Fix roadmap indentation --- docs/roadmap.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/roadmap.rst b/docs/roadmap.rst index 35a581c..6fca7b7 100644 --- a/docs/roadmap.rst +++ b/docs/roadmap.rst @@ -46,11 +46,11 @@ situation. as in an installation/removal context - New subcommand: resolve-dependents to find all dependents in an *index* - USER GUIDES -- How to use repositories as generic installation trackers -- How to track dependencies between kubernetes services -- How to track dependencies between cross-language builds and use this for - that + - How to use repositories as generic installation trackers + - How to track dependencies between kubernetes services + - How to track dependencies between cross-language builds and use this for + that - Documentation and/or code on the topic of supporting the use case of different architectures of the same package using prioritized indexes of packages named the same with different contents. From 356d5e3e397af5afb514dfa2c937963e4e6af1f9 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sun, 17 Nov 2019 14:00:16 -0700 Subject: [PATCH 29/31] Fix whitespace in roadmap --- docs/roadmap.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/roadmap.rst b/docs/roadmap.rst index 6fca7b7..36885a7 100644 --- a/docs/roadmap.rst +++ b/docs/roadmap.rst @@ -10,7 +10,7 @@ Future Releases --------------- - Tutorial-like help screens designed to keep people from needing to switch - from docs to cli and back. + from docs to cli and back. - Shortened versions of all subcommands, including documentation updates. From ef7e8a47008f4d2347dc967eed16d3b24fa20252 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sun, 17 Nov 2019 14:01:31 -0700 Subject: [PATCH 30/31] Fix roadmap --- docs/roadmap.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/roadmap.rst b/docs/roadmap.rst index 36885a7..855fc05 100644 --- a/docs/roadmap.rst +++ b/docs/roadmap.rst @@ -31,7 +31,6 @@ DevOps professionals, for whom we build this tool, need to a dependency management tool that can get them out of dependency hell no matter what their situation. -- Shortened versions of all subcommands, including documentation updates. - The ability to slurp from JDBC URLs for indexes - An extension will be made to ensure that username and password From d2eea71b1538be142f38c53207d05a5ab422e809 Mon Sep 17 00:00:00 2001 From: Daniel Jay Haskin Date: Sun, 17 Nov 2019 14:02:12 -0700 Subject: [PATCH 31/31] Fix copyright statement --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 71a8fa7..b3e00d6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,7 +50,7 @@ # General information about the project. project = 'Degasolv' -copyright = '2017-2018, Daniel Jay Haskin et. al., see all authors under "Authors and Contributions"' +copyright = '2017-2019, Daniel Jay Haskin et. al., see all authors under "Authors and Contributions"' author = 'Daniel Jay Haskin et. al., see the authors under "Authors and Contributions"' # The version info for the project you're documenting, acts as replacement for