diff --git a/Makefile b/Makefile index 02752599aba..5e1cadc49bc 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ NUM_PROC = $(nproc --all) ERA ?= bage PROFILE ?= default-${ERA} +BACKEND ?= supervisor REV ?= master ITER ?= ARGS ?= @@ -49,7 +50,7 @@ ci-targets: $(CI_TARGETS) ## Base targets: ## shell: ## Nix shell, (workbench from /nix/store), vars: PROFILE, CMD, RUN - nix-shell -A 'workbench-shell' --max-jobs 8 --cores 0 --show-trace --argstr profileName ${PROFILE} ${ARGS} ${if ${CMD},--command "${CMD}"} ${if ${RUN},--run "${RUN}"} + nix-shell -A 'workbench-shell' --max-jobs 8 --cores 0 --show-trace --argstr profileName ${PROFILE} --argstr backendName ${BACKEND} ${ARGS} ${if ${CMD},--command "${CMD}"} ${if ${RUN},--run "${RUN}"} shell-dev shell-prof shell-nix: shell shell-nix: ARGS += --arg 'workbenchDevMode' false ## Nix shell, (workbench from Nix store), vars: PROFILE, CMD, RUN shell-prof: ARGS += --arg 'profiled' true ## Nix shell, everything Haskell built profiled diff --git a/flake.lock b/flake.lock index 1a6fa0b7eeb..944c83de1a6 100644 --- a/flake.lock +++ b/flake.lock @@ -1509,7 +1509,7 @@ }, "cardano-mainnet-mirror_10": { "inputs": { - "nixpkgs": "nixpkgs_14" + "nixpkgs": "nixpkgs_15" }, "locked": { "lastModified": 1642701714, @@ -1528,7 +1528,7 @@ }, "cardano-mainnet-mirror_11": { "inputs": { - "nixpkgs": "nixpkgs_15" + "nixpkgs": "nixpkgs_16" }, "locked": { "lastModified": 1642701714, @@ -1547,7 +1547,7 @@ }, "cardano-mainnet-mirror_12": { "inputs": { - "nixpkgs": "nixpkgs_16" + "nixpkgs": "nixpkgs_17" }, "locked": { "lastModified": 1642701714, @@ -1566,7 +1566,7 @@ }, "cardano-mainnet-mirror_13": { "inputs": { - "nixpkgs": "nixpkgs_17" + "nixpkgs": "nixpkgs_18" }, "locked": { "lastModified": 1642701714, @@ -1585,7 +1585,7 @@ }, "cardano-mainnet-mirror_14": { "inputs": { - "nixpkgs": "nixpkgs_18" + "nixpkgs": "nixpkgs_19" }, "locked": { "lastModified": 1642701714, @@ -1604,7 +1604,7 @@ }, "cardano-mainnet-mirror_15": { "inputs": { - "nixpkgs": "nixpkgs_19" + "nixpkgs": "nixpkgs_20" }, "locked": { "lastModified": 1642701714, @@ -1623,7 +1623,7 @@ }, "cardano-mainnet-mirror_2": { "inputs": { - "nixpkgs": "nixpkgs_3" + "nixpkgs": "nixpkgs_4" }, "locked": { "lastModified": 1642701714, @@ -1642,7 +1642,7 @@ }, "cardano-mainnet-mirror_3": { "inputs": { - "nixpkgs": "nixpkgs_6" + "nixpkgs": "nixpkgs_7" }, "locked": { "lastModified": 1642701714, @@ -1661,7 +1661,7 @@ }, "cardano-mainnet-mirror_4": { "inputs": { - "nixpkgs": "nixpkgs_8" + "nixpkgs": "nixpkgs_9" }, "locked": { "lastModified": 1642701714, @@ -1680,7 +1680,7 @@ }, "cardano-mainnet-mirror_5": { "inputs": { - "nixpkgs": "nixpkgs_9" + "nixpkgs": "nixpkgs_10" }, "locked": { "lastModified": 1642701714, @@ -1699,7 +1699,7 @@ }, "cardano-mainnet-mirror_6": { "inputs": { - "nixpkgs": "nixpkgs_10" + "nixpkgs": "nixpkgs_11" }, "locked": { "lastModified": 1642701714, @@ -1718,7 +1718,7 @@ }, "cardano-mainnet-mirror_7": { "inputs": { - "nixpkgs": "nixpkgs_11" + "nixpkgs": "nixpkgs_12" }, "locked": { "lastModified": 1642701714, @@ -1737,7 +1737,7 @@ }, "cardano-mainnet-mirror_8": { "inputs": { - "nixpkgs": "nixpkgs_12" + "nixpkgs": "nixpkgs_13" }, "locked": { "lastModified": 1642701714, @@ -1756,7 +1756,7 @@ }, "cardano-mainnet-mirror_9": { "inputs": { - "nixpkgs": "nixpkgs_13" + "nixpkgs": "nixpkgs_14" }, "locked": { "lastModified": 1642701714, @@ -3356,6 +3356,21 @@ } }, "flake-utils_24": { + "locked": { + "lastModified": 1623875721, + "narHash": "sha256-A8BU7bjS5GirpAUv4QA+QnJ4CceLHkcXdRp4xITDB0s=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "f7e004a55b120c02ecb6219596820fcd32ca8772", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_25": { "locked": { "lastModified": 1653893745, "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", @@ -3370,7 +3385,7 @@ "type": "github" } }, - "flake-utils_25": { + "flake-utils_26": { "locked": { "lastModified": 1659877975, "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", @@ -3385,7 +3400,7 @@ "type": "github" } }, - "flake-utils_26": { + "flake-utils_27": { "locked": { "lastModified": 1653893745, "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", @@ -3402,11 +3417,11 @@ }, "flake-utils_3": { "locked": { - "lastModified": 1644229661, - "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", + "lastModified": 1653893745, + "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "3cecb5b042f7f209c56ffd8371b2711a290ec797", + "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", "type": "github" }, "original": { @@ -3447,11 +3462,11 @@ }, "flake-utils_6": { "locked": { - "lastModified": 1623875721, - "narHash": "sha256-A8BU7bjS5GirpAUv4QA+QnJ4CceLHkcXdRp4xITDB0s=", + "lastModified": 1644229661, + "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", "owner": "numtide", "repo": "flake-utils", - "rev": "f7e004a55b120c02ecb6219596820fcd32ca8772", + "rev": "3cecb5b042f7f209c56ffd8371b2711a290ec797", "type": "github" }, "original": { @@ -3898,7 +3913,7 @@ }, "gomod2nix": { "inputs": { - "nixpkgs": "nixpkgs_20", + "nixpkgs": "nixpkgs_21", "utils": "utils_23" }, "locked": { @@ -4326,7 +4341,7 @@ "cabal-32": "cabal-32_10", "cabal-34": "cabal-34_10", "cardano-shell": "cardano-shell_10", - "flake-utils": "flake-utils_10", + "flake-utils": "flake-utils_11", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_10", "hackage": "hackage_8", "hpc-coveralls": "hpc-coveralls_10", @@ -4368,7 +4383,7 @@ "cabal-34": "cabal-34_11", "cabal-36": "cabal-36_10", "cardano-shell": "cardano-shell_11", - "flake-utils": "flake-utils_11", + "flake-utils": "flake-utils_12", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_11", "hackage": "hackage_9", "hpc-coveralls": "hpc-coveralls_11", @@ -4407,7 +4422,7 @@ "cabal-34": "cabal-34_12", "cabal-36": "cabal-36_11", "cardano-shell": "cardano-shell_12", - "flake-utils": "flake-utils_12", + "flake-utils": "flake-utils_13", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_12", "hackage": "hackage_10", "hpc-coveralls": "hpc-coveralls_12", @@ -4448,7 +4463,7 @@ "cabal-34": "cabal-34_13", "cabal-36": "cabal-36_12", "cardano-shell": "cardano-shell_13", - "flake-utils": "flake-utils_13", + "flake-utils": "flake-utils_14", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_13", "hackage": "hackage_11", "hpc-coveralls": "hpc-coveralls_13", @@ -4490,7 +4505,7 @@ "cabal-32": "cabal-32_14", "cabal-34": "cabal-34_14", "cardano-shell": "cardano-shell_14", - "flake-utils": "flake-utils_14", + "flake-utils": "flake-utils_15", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_14", "hackage": "hackage_12", "hpc-coveralls": "hpc-coveralls_14", @@ -4532,7 +4547,7 @@ "cabal-34": "cabal-34_15", "cabal-36": "cabal-36_13", "cardano-shell": "cardano-shell_15", - "flake-utils": "flake-utils_15", + "flake-utils": "flake-utils_16", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_15", "hackage": "hackage_13", "hpc-coveralls": "hpc-coveralls_15", @@ -4571,7 +4586,7 @@ "cabal-34": "cabal-34_16", "cabal-36": "cabal-36_14", "cardano-shell": "cardano-shell_16", - "flake-utils": "flake-utils_16", + "flake-utils": "flake-utils_17", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_16", "hackage": "hackage_14", "hpc-coveralls": "hpc-coveralls_16", @@ -4611,7 +4626,7 @@ "cabal-32": "cabal-32_17", "cabal-34": "cabal-34_17", "cardano-shell": "cardano-shell_17", - "flake-utils": "flake-utils_17", + "flake-utils": "flake-utils_18", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_17", "hackage": "hackage_15", "hpc-coveralls": "hpc-coveralls_17", @@ -4651,7 +4666,7 @@ "cabal-34": "cabal-34_18", "cabal-36": "cabal-36_15", "cardano-shell": "cardano-shell_18", - "flake-utils": "flake-utils_18", + "flake-utils": "flake-utils_19", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_18", "hackage": "hackage_16", "hpc-coveralls": "hpc-coveralls_18", @@ -4689,7 +4704,7 @@ "cabal-34": "cabal-34_19", "cabal-36": "cabal-36_16", "cardano-shell": "cardano-shell_19", - "flake-utils": "flake-utils_19", + "flake-utils": "flake-utils_20", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_19", "hackage": "hackage_17", "hpc-coveralls": "hpc-coveralls_19", @@ -4767,7 +4782,7 @@ "cabal-32": "cabal-32_20", "cabal-34": "cabal-34_20", "cardano-shell": "cardano-shell_20", - "flake-utils": "flake-utils_20", + "flake-utils": "flake-utils_21", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_20", "hackage": "hackage_18", "hpc-coveralls": "hpc-coveralls_20", @@ -4806,7 +4821,7 @@ "cabal-34": "cabal-34_21", "cabal-36": "cabal-36_17", "cardano-shell": "cardano-shell_21", - "flake-utils": "flake-utils_21", + "flake-utils": "flake-utils_22", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_21", "hackage": "hackage_19", "hpc-coveralls": "hpc-coveralls_21", @@ -4843,7 +4858,7 @@ "cabal-34": "cabal-34_22", "cabal-36": "cabal-36_18", "cardano-shell": "cardano-shell_22", - "flake-utils": "flake-utils_22", + "flake-utils": "flake-utils_23", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_22", "hackage": "hackage_20", "hpc-coveralls": "hpc-coveralls_22", @@ -4881,7 +4896,7 @@ "cabal-32": "cabal-32_23", "cabal-34": "cabal-34_23", "cardano-shell": "cardano-shell_23", - "flake-utils": "flake-utils_23", + "flake-utils": "flake-utils_24", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_23", "hackage": "hackage_21", "hpc-coveralls": "hpc-coveralls_23", @@ -4919,7 +4934,7 @@ "cabal-34": "cabal-34_3", "cabal-36": "cabal-36_3", "cardano-shell": "cardano-shell_3", - "flake-utils": "flake-utils_3", + "flake-utils": "flake-utils_4", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_3", "hackage": "hackage_2", "hpc-coveralls": "hpc-coveralls_3", @@ -4958,7 +4973,7 @@ "cabal-34": "cabal-34_4", "cabal-36": "cabal-36_4", "cardano-shell": "cardano-shell_4", - "flake-utils": "flake-utils_4", + "flake-utils": "flake-utils_5", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_4", "hackage": [ "node-measured", @@ -4999,7 +5014,7 @@ "cabal-34": "cabal-34_5", "cabal-36": "cabal-36_5", "cardano-shell": "cardano-shell_5", - "flake-utils": "flake-utils_5", + "flake-utils": "flake-utils_6", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_5", "hackage": "hackage_3", "hpc-coveralls": "hpc-coveralls_5", @@ -5039,7 +5054,7 @@ "cabal-34": "cabal-34_6", "cabal-36": "cabal-36_6", "cardano-shell": "cardano-shell_6", - "flake-utils": "flake-utils_6", + "flake-utils": "flake-utils_7", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_6", "hackage": "hackage_4", "hpc-coveralls": "hpc-coveralls_6", @@ -5077,7 +5092,7 @@ "cabal-34": "cabal-34_7", "cabal-36": "cabal-36_7", "cardano-shell": "cardano-shell_7", - "flake-utils": "flake-utils_7", + "flake-utils": "flake-utils_8", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_7", "hackage": "hackage_5", "hpc-coveralls": "hpc-coveralls_7", @@ -5116,7 +5131,7 @@ "cabal-34": "cabal-34_8", "cabal-36": "cabal-36_8", "cardano-shell": "cardano-shell_8", - "flake-utils": "flake-utils_8", + "flake-utils": "flake-utils_9", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_8", "hackage": "hackage_6", "hpc-coveralls": "hpc-coveralls_8", @@ -5157,7 +5172,7 @@ "cabal-34": "cabal-34_9", "cabal-36": "cabal-36_9", "cardano-shell": "cardano-shell_9", - "flake-utils": "flake-utils_9", + "flake-utils": "flake-utils_10", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk_9", "hackage": "hackage_7", "hpc-coveralls": "hpc-coveralls_9", @@ -6823,7 +6838,7 @@ }, "n2c": { "inputs": { - "flake-utils": "flake-utils_26", + "flake-utils": "flake-utils_27", "nixpkgs": [ "tullia", "std", @@ -7267,8 +7282,27 @@ }, "nix2container": { "inputs": { - "flake-utils": "flake-utils_24", - "nixpkgs": "nixpkgs_21" + "flake-utils": "flake-utils_3", + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1665039323, + "narHash": "sha256-SAh3ZjFGsaCI8FRzXQyp56qcGdAqgKEfJWPCQ0Sr7tQ=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "b008fe329ffb59b67bf9e7b08ede6ee792f2741a", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, + "nix2container_2": { + "inputs": { + "flake-utils": "flake-utils_25", + "nixpkgs": "nixpkgs_22" }, "locked": { "lastModified": 1658567952, @@ -7319,7 +7353,7 @@ "nix_2": { "inputs": { "lowdown-src": "lowdown-src_2", - "nixpkgs": "nixpkgs_4", + "nixpkgs": "nixpkgs_5", "nixpkgs-regression": "nixpkgs-regression_2" }, "locked": { @@ -7340,7 +7374,7 @@ "nix_3": { "inputs": { "lowdown-src": "lowdown-src_3", - "nixpkgs": "nixpkgs_5", + "nixpkgs": "nixpkgs_6", "nixpkgs-regression": "nixpkgs-regression_3" }, "locked": { @@ -7361,7 +7395,7 @@ "nix_4": { "inputs": { "lowdown-src": "lowdown-src_4", - "nixpkgs": "nixpkgs_7", + "nixpkgs": "nixpkgs_8", "nixpkgs-regression": "nixpkgs-regression_4" }, "locked": { @@ -9113,6 +9147,20 @@ } }, "nixpkgs_20": { + "locked": { + "lastModified": 1642336556, + "narHash": "sha256-QSPPbFEwy0T0DrIuSzAACkaANPQaR1lZR/nHZGz9z04=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f3d9d4bd898cca7d04af2ae4f6ef01f2219df3d6", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_21": { "locked": { "lastModified": 1653581809, "narHash": "sha256-Uvka0V5MTGbeOfWte25+tfRL3moECDh1VwokWSZUdoY=", @@ -9128,7 +9176,7 @@ "type": "github" } }, - "nixpkgs_21": { + "nixpkgs_22": { "locked": { "lastModified": 1654807842, "narHash": "sha256-ADymZpr6LuTEBXcy6RtFHcUZdjKTBRTMYwu19WOx17E=", @@ -9143,7 +9191,7 @@ "type": "github" } }, - "nixpkgs_22": { + "nixpkgs_23": { "locked": { "lastModified": 1665087388, "narHash": "sha256-FZFPuW9NWHJteATOf79rZfwfRn5fE0wi9kRzvGfDHPA=", @@ -9160,6 +9208,21 @@ } }, "nixpkgs_3": { + "locked": { + "lastModified": 1654807842, + "narHash": "sha256-ADymZpr6LuTEBXcy6RtFHcUZdjKTBRTMYwu19WOx17E=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "fc909087cc3386955f21b4665731dbdaceefb1d8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { "locked": { "lastModified": 1642336556, "narHash": "sha256-QSPPbFEwy0T0DrIuSzAACkaANPQaR1lZR/nHZGz9z04=", @@ -9173,7 +9236,7 @@ "type": "indirect" } }, - "nixpkgs_4": { + "nixpkgs_5": { "locked": { "lastModified": 1632864508, "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=", @@ -9188,7 +9251,7 @@ "type": "indirect" } }, - "nixpkgs_5": { + "nixpkgs_6": { "locked": { "lastModified": 1632864508, "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=", @@ -9203,7 +9266,7 @@ "type": "indirect" } }, - "nixpkgs_6": { + "nixpkgs_7": { "locked": { "lastModified": 1642336556, "narHash": "sha256-QSPPbFEwy0T0DrIuSzAACkaANPQaR1lZR/nHZGz9z04=", @@ -9217,7 +9280,7 @@ "type": "indirect" } }, - "nixpkgs_7": { + "nixpkgs_8": { "locked": { "lastModified": 1632864508, "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=", @@ -9232,20 +9295,6 @@ "type": "indirect" } }, - "nixpkgs_8": { - "locked": { - "lastModified": 1642336556, - "narHash": "sha256-QSPPbFEwy0T0DrIuSzAACkaANPQaR1lZR/nHZGz9z04=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "f3d9d4bd898cca7d04af2ae4f6ef01f2219df3d6", - "type": "github" - }, - "original": { - "id": "nixpkgs", - "type": "indirect" - } - }, "nixpkgs_9": { "locked": { "lastModified": 1642336556, @@ -10413,6 +10462,7 @@ "nixpkgs" ], "iohkNix": "iohkNix_2", + "nix2container": "nix2container", "nixTools": "nixTools", "nixpkgs": [ "haskellNix", @@ -10799,7 +10849,7 @@ "blank": "blank", "devshell": "devshell", "dmerge": "dmerge", - "flake-utils": "flake-utils_25", + "flake-utils": "flake-utils_26", "makes": [ "tullia", "std", @@ -10813,7 +10863,7 @@ ], "n2c": "n2c", "nixago": "nixago", - "nixpkgs": "nixpkgs_22", + "nixpkgs": "nixpkgs_23", "yants": "yants" }, "locked": { @@ -10833,7 +10883,7 @@ "tullia": { "inputs": { "nix-nomad": "nix-nomad", - "nix2container": "nix2container", + "nix2container": "nix2container_2", "nixpkgs": [ "nixpkgs" ], diff --git a/flake.nix b/flake.nix index c5e5c38c9ab..4e7d0c8b66e 100644 --- a/flake.nix +++ b/flake.nix @@ -73,6 +73,8 @@ url = "github:input-output-hk/tullia"; inputs.nixpkgs.follows = "nixpkgs"; }; + + nix2container.url = "github:nlewo/nix2container"; }; outputs = @@ -90,6 +92,7 @@ , node-process , cardano-node-workbench , tullia + , nix2container , ... }@input: let @@ -114,7 +117,7 @@ iohkNix.overlays.cardano-lib iohkNix.overlays.utils (final: prev: { - inherit customConfig; + inherit customConfig nix2container; gitrev = final.customConfig.gitrev or self.rev or "0000000000000000000000000000000000000000"; commonLib = lib // iohkNix.lib diff --git a/nix/custom-config.nix b/nix/custom-config.nix index 83d89dfa949..d06a2fa1635 100644 --- a/nix/custom-config.nix +++ b/nix/custom-config.nix @@ -5,6 +5,7 @@ self: { stateDir = "run/current"; batchName = "plain"; profileName = "default-bage"; + backendName = "supervisor"; basePort = 30000; enableEKG = true; workbenchDevMode = true; diff --git a/nix/pkgs.nix b/nix/pkgs.nix index 76d5b443fff..f0005afdcad 100644 --- a/nix/pkgs.nix +++ b/nix/pkgs.nix @@ -84,6 +84,9 @@ final: prev: with final; { supervisord-workbench-nix = { workbench ? pkgs.workbench, ... }@args: pkgs.callPackage ./workbench/backend/supervisor.nix args; + nomad-workbench = + { workbench ? pkgs.workbench, ... }@args: pkgs.callPackage ./workbench/backend/nomad.nix (args // { inherit nix2container; }); + all-profiles-json = (workbench.all-profiles{ inherit (supervisord-workbench-nix) backend; }).JSON; # An instance of the workbench, specialised to the supervisord backend and a profile, @@ -103,6 +106,20 @@ final: prev: with final; { inherit batchName profileName supervisord-workbench cardano-node-rev; }; + nomad-workbench-for-profile = + { batchName ? customConfig.localCluster.batchName + , profileName ? customConfig.localCluster.profileName + , useCabalRun ? false + , workbenchDevMode ? false + , profiled ? false + , nomad-workbench ? pkgs.callPackage ./workbench/backend/nomad.nix { inherit nix2container; } + , cardano-node-rev ? null + }: + pkgs.callPackage ./workbench/backend/nomad-run.nix + { + inherit batchName profileName nomad-workbench cardano-node-rev; + }; + # Disable failing python uvloop tests python38 = prev.python38.override { packageOverrides = pythonFinal: pythonPrev: { diff --git a/nix/workbench/backend/nomad-conf.nix b/nix/workbench/backend/nomad-conf.nix new file mode 100644 index 00000000000..614fa7f7cdf --- /dev/null +++ b/nix/workbench/backend/nomad-conf.nix @@ -0,0 +1,81 @@ +{ pkgs +# Cardano packages/executables. +, cardano-node, cardano-tracer, tx-generator +# OCI Image builder. +, nix2container +}: + +let + + # Why `nix2container` instead of the built-in `dockerTools` ?: + # - https://lewo.abesis.fr/posts/nix-build-container-image/ + # - https://discourse.nixos.org/t/nix2container-another-dockertools-buildimage-implementation-based-on-skopeo/21688 + n2c = nix2container.outputs.packages.x86_64-linux.nix2container; + + clusterImage = n2c.buildImage { + name = "registry.workbench.iog.io/cluster"; + # Adds `/etc/protocols` and ``/etc/services` to the root directory. + # FIXME: Inside the container still can't resolve `localhost` but can + # resolve WAN domains using public DNS servers. + # Running `bash-5.1# /nix/store/*-glibc-#-bin/bin/getent hosts localhost` + # inside the container returns nothing and python stuff like `supervisord` + # breaks: "error: , [Errno -2] Name or service not known: file: /nix/store/hb1lzaisgx2m9n29hqhh6yp6hasplq1v-python3-3.9.10/lib/python3.9/socket.py line: 954" + # Further reading for hints: + # https://stackoverflow.com/questions/39965432/docker-container-unable-to-resolve-localhost + copyToRoot = with pkgs; [ iana-etc ]; + # All these layers are added to /nix/store, nothing is in `$PATH`. + maxLayers = 25; + layers = with pkgs; [ + # Runtime to be able run bash commands from `podman`/`nomad`. + (n2c.buildLayer {deps = [ bashInteractive coreutils ];}) + # Supervisor. + (n2c.buildLayer {deps = [ python3Packages.supervisor ];}) + # Cardano packages. + (n2c.buildLayer {deps = [ cardano-node ];}) + (n2c.buildLayer {deps = [ cardano-tracer ];}) + (n2c.buildLayer {deps = [ tx-generator ];}) + ]; + # OCI container specification: + # https://github.com/opencontainers/image-spec/blob/3a7f492d3f1bcada656a7d8c08f3f9bbd05e7406/specs-go/v1/config.go#L24 + config = { + # Volumes are mounted as user `0:0`, I have no choice here. + User = "0:0"; + # The stanza `WorkingDir` is not used because the config file of + # `supervisord` depends on the working directory. + Entrypoint = + let + entrypoint = pkgs.writeShellApplication { + name = "entrypoint"; + runtimeInputs = with pkgs; [ + coreutils + bashInteractive + python3Packages.supervisor + ]; + text = '' + # The SUPERVISOR_NIX variable must be set + [ -z "''${SUPERVISOR_NIX:-}" ] && echo "SUPERVISOR_NIX env var must be set -- aborting" && exit 1 + + # The SUPERVISORD_CONFIG variable must be set + [ -z "''${SUPERVISORD_CONFIG:-}" ] && echo "SUPERVISORD_CONFIG env var must be set -- aborting" && exit 1 + + # Create a link to the `supervisor` Nix folder. + # First check if already exists to be able to restart containers. + if ! test -e "$SUPERVISOR_NIX" + then + "${pkgs.coreutils}"/bin/ln -s "${pkgs.python3Packages.supervisor}" "$SUPERVISOR_NIX" + fi + + # Start `supervisord` on the foreground. + "${pkgs.python3Packages.supervisor}"/bin/supervisord --nodaemon --configuration "$SUPERVISORD_CONFIG" + ''; + }; + in + [ "${entrypoint}/bin/entrypoint" ]; + }; + }; + +in { + + inherit clusterImage; + +} diff --git a/nix/workbench/backend/nomad-run.nix b/nix/workbench/backend/nomad-run.nix new file mode 100644 index 00000000000..534667eaff1 --- /dev/null +++ b/nix/workbench/backend/nomad-run.nix @@ -0,0 +1,152 @@ +let + batchNameDefault = "plain"; + profileNameDefault = "default-bage"; +in +{ pkgs +, cardanoNodePackages +, nomad-workbench +## +, profileName ? profileNameDefault +, batchName ? batchNameDefault +## +, workbenchDevMode ? false +, cardano-node-rev ? "0000000000000000000000000000000000000000" +}: +let + inherit (nomad-workbench) workbench backend cacheDir stateDir basePort; + + with-nomad-profile = + { envArgsOverride ? {} }: ## TODO: envArgsOverride is not used! + workbench.with-profile + { inherit backend profileName; }; + + inherit (with-nomad-profile {}) profileNix profile topology genesis; +in + let + + inherit (profile.value) era composition monetary; + + path = pkgs.lib.makeBinPath path'; + path' = + [ cardanoNodePackages.bech32 pkgs.jq pkgs.gnused pkgs.coreutils pkgs.bash pkgs.moreutils + ] + ## In dev mode, call the script directly: + ++ pkgs.lib.optionals (!workbenchDevMode) + [ workbench.workbench ]; + + interactive-start = pkgs.writeScriptBin "start-cluster" '' + set -euo pipefail + + export PATH=$PATH:${path} + unset WB_MODE_CABAL= + wb start \ + --batch-name ${batchName} \ + --profile-name ${profileName} \ + --profile ${profile} \ + --cache-dir ${cacheDir} \ + --base-port ${toString basePort} \ + ''${WB_MODE_CABAL:+--cabal} \ + "$@" + ''; + + interactive-stop = pkgs.writeScriptBin "stop-cluster" '' + set -euo pipefail + + wb finish "$@" + ''; + + interactive-restart = pkgs.writeScriptBin "restart-cluster" '' + set -euo pipefail + + wb run restart "$@" && \ + echo "workbench: alternate command for this action: wb run restart" >&2 + ''; + + nodeBuildProduct = + name: + "report ${name}-log $out ${name}/stdout"; + + profile-run = + { trace ? false }: + let + inherit + (with-nomad-profile + { envArgsOverride = { cacheDir = "./cache"; stateDir = "./"; }; }) + profileNix profile topology genesis; + + run = pkgs.runCommand "workbench-run-nomad-${profileName}" + { requiredSystemFeatures = [ "benchmark" ]; + nativeBuildInputs = with cardanoNodePackages; with pkgs; [ + bash + bech32 + coreutils + gnused + jq + moreutils + nixWrapped + pstree +# TODO: nomad + workbench.workbench + zstd + ]; + } + '' + mkdir -p $out/{cache,nix-support} + cd $out + export HOME=$out + + export WB_BACKEND=nomad + export CARDANO_NODE_SOCKET_PATH=$(wb backend get-node-socket-path ${stateDir} node-0) + + cmd=( + wb + ${pkgs.lib.optionalString trace "--trace"} + start + --profile-name ${profileName} + --profile ${profile} + --topology ${topology} + --genesis-cache-entry ${genesis} + --batch-name smoke-test + --base-port ${toString basePort} + --node-source ${cardanoNodePackages.cardano-node.src.origSrc} + --node-rev ${cardano-node-rev} + --cache-dir ./cache + ) + echo "''${cmd[*]}" > $out/wb-start.sh + + time "''${cmd[@]}" 2>&1 | + tee $out/wb-start.log + + ## Convert structure from $out/run/RUN-ID/* to $out/*: + rm -rf cache + rm -f run/{current,-current} + find $out -type s | xargs rm -f + run=$(cd run; ls) + (cd run; tar c $run --zstd) > archive.tar.zst + mv run/$run/* . + rmdir run/$run run + + cat > $out/nix-support/hydra-build-products < $out/clusterImageName + echo $clusterImageTag > $out/clusterImageTag + ln -s $clusterImageCopyToPodman/bin/copy-to-podman $out/clusterImageCopyToPodman + ''; + }; +in +{ + inherit cacheDir stateDir basePort; + inherit workbench; + inherit backend; +} diff --git a/nix/workbench/backend/nomad.sh b/nix/workbench/backend/nomad.sh new file mode 100644 index 00000000000..8de6890901f --- /dev/null +++ b/nix/workbench/backend/nomad.sh @@ -0,0 +1,662 @@ +usage_nomad() { + usage "nomad" "Backend: manages a local cluster using 'nomad' (and 'podman')" </dev/null | grep ':30000 ' | wc -l)" != "0" + ;; + + setenv-defaults ) + local usage="USAGE: wb nomad $op PROFILE-DIR" + local profile_dir=${1:?$usage} + + # Look up `supervisord` config file produced by Nix (run profile). + setenvjqstr 'supervisord_conf' "$profile_dir"/supervisor.conf + # The `--serverurl` argument is needed in every call to `nomad exec`. + # The problem is that if we use "127.0.0.1:9001" as parameter (without + # the "http" part) the container returns: + # error: , Unknown protocol for serverurl 127.0.0.1:9001: file: /nix/store/izqhlj5i1x9ldyn43d02kcy4mafmj3ci-python3.9-supervisor-4.2.4/lib/python3.9/site-packages/supervisor/xmlrpc.py line: 508 + # Without using the `--serverurl` parameter at all (using INI config + # file's [inet_http_server] port stanza) also without "http://": + # error: , [Errno -2] Name or service not known: file: /nix/store/hb1lzaisgx2m9n29hqhh6yp6hasplq1v-python3-3.9.10/lib/python3.9/socket.py line: 954 + # If I add the "http" part to the INI file, when starting `supervisord` + # inside the container I get (from journald): + # Nov 02 11:44:36 hostname cluster-18f3852f-e067-6394-8159-66a7b8da2ecc[1088457]: Error: Cannot open an HTTP server: socket.error reported -2 + # Nov 02 11:44:36 hostname cluster-18f3852f-e067-6394-8159-66a7b8da2ecc[1088457]: For help, use /nix/store/izqhlj5i1x9ldyn43d02kcy4mafmj3ci-python3.9-supervisor-4.2.4/bin/supervisord -h + setenvjqstr 'supervisord_url' "unix:///tmp/supervisor.sock" + # Look up `cluster` OCI image's name and tag (also Nix profile). + setenvjqstr 'oci_image_name' ${WB_OCI_IMAGE_NAME:-$(cat "$profile_dir/clusterImageName")} + setenvjqstr 'oci_image_tag' ${WB_OCI_IMAGE_TAG:-$(cat "$profile_dir/clusterImageTag")} + # Script that creates the OCI image from nix2container layered output. + setenvjqstr 'oci_image_skopeo_script' "$profile_dir/clusterImageCopyToPodman" + # Set cluster's podman container defaults. + # The workbench is expecting an specific hierarchy of folders and files. + setenvjqstr 'container_workdir' "/tmp/cluster/" + setenvjqstr 'container_mountpoint' "/tmp/cluster/run/current" + # The `supervisord` binary is installed inside the container but not + # added to $PATH (resides in /nix/store), so a desired location is + # passed to the container as an environment variable to create a symlink + # to it. + setenvjqstr 'container_supervisor_nix' "/tmp/cluster/run/current/supervisor/nix-store" + # The container need to know where `supervisord` config file is located + # so it can be started. This is passed as an environment variable. + setenvjqstr 'container_supervisord_conf' "/tmp/cluster/run/current/supervisor/supervisord.conf" + ;; + + # Man pages for Podman configuration files: + # https://man.archlinux.org/man/community/podman/podman.1.en + # https://man.archlinux.org/man/containers.conf.5 + # https://man.archlinux.org/man/containers-storage.conf.5 + # https://man.archlinux.org/man/containers-policy.json.5 + + allocate-run ) + local usage="USAGE: wb nomad $op RUN-DIR" + local dir=${1:?$usage}; shift + + while test $# -gt 0 + do case "$1" in + --* ) msg "FATAL: unknown flag '$1'"; usage_docker;; + * ) break;; esac; shift; done + + # The `genesis/utxo-keys` directory is used as a volume for the + # `generator` service but it's not always present/created. + if ! test -e "$dir"/genesis/utxo-keys + then + mkdir -p "$dir"/genesis/utxo-keys + else + # HACK: UGLY!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ############### FIXME: Fix it in `genesis.sh` ############### + mv "$dir"/genesis/utxo-keys "$dir"/genesis/utxo-keys.bak + # The `genesis/utxo-keys` directory is used as a volume for the + # `generator` service but it's not always present/created. + mkdir -p "$dir"/genesis/utxo-keys + cp -r "$dir"/genesis/utxo-keys.bak/* "$dir"/genesis/utxo-keys/ + fi + + # Populate the files needed by the `supervisord` instance running inside + # the container. + local supervisord_conf=$(envjqr 'supervisord_conf') + mkdir -p "$dir"/supervisor + # If $dir is being mounted inside the container the file must be copied + # because if it references something outside the container's mounted + # volume the container probably won't be able to access it. + cp -f "$supervisord_conf" "$dir"/supervisor/supervisord.conf + + # Create the "cluster" OCI image. + local oci_image_name=$( envjqr 'oci_image_name') + local oci_image_tag=$( envjqr 'oci_image_tag') + local oci_image_skopeo_script=$(envjqr 'oci_image_skopeo_script') + msg "Creating OCI image ..." + # Forced the `overlay` storage driver or podman won't see the image. + # https://docs.podman.io/en/latest/markdown/podman.1.html#note-unsupported-file-systems-in-rootless-mode + STORAGE_DRIVER=overlay "$oci_image_skopeo_script" + # Check that `podman` can see the "cluster" OCI image. + if ! podman image exists "${oci_image_name}:${oci_image_tag}" + then + fatal "OCI image ${oci_image_name}:${oci_image_tag} cannot be found by podman" + else + msg "OCI image named \"${oci_image_name}:${oci_image_tag}\" created" + fi + + # Configure `nomad` and the `podman` plugin/task driver. + nomad_create_folders_and_config "$dir" + msg "Preparing podman API service for nomad driver \`nomad-driver-podman\` ..." + nomad_start_podman_service "$dir" + + # Start `nomad` agent in "-dev-` mode`". + msg "Starting nomad agent ..." + # The Nomad agent is a long running process which runs on every machine + # that is part of the Nomad cluster. The behavior of the agent depends + # on if it is running in client or server mode. Clients are responsible + # for running tasks, while servers are responsible for managing the + # cluster. + # -dev: Start the agent in development mode. This enables a + # pre-configured dual-role agent (client + server) which is useful for + # developing or testing Nomad. No other configuration is required to + # start the agent in this mode, but you may pass an optional + # comma-separated list of mode configurations + nomad agent -config="$dir/nomad/config" -dev -log-level=INFO >> "$dir/nomad/stdout" 2>> "$dir/nomad/stderr" & + echo "$!" > "$dir/nomad/nomad.pid" + setenvjqstr 'nomad_pid' $(cat $dir/nomad/nomad.pid) + msg "Nomad started with PID $(cat $dir/nomad/nomad.pid)" + + # Wait for nomad agent: + msg "Waiting for the listening HTTP server ..." + local i=0 + local patience=25 + until curl -Isf 127.0.0.1:4646 2>&1 | head --lines=1 | grep --quiet "HTTP/1.1" + do printf "%3d" $i; sleep 1 + i=$((i+1)) + if test $i -ge $patience + then echo + progress "nomad agent" "$(red FATAL): workbench: nomad agent: patience ran out after ${patience}s, 127.0.0.1:4646" + backend_nomad stop-cluster "$dir" + fatal "nomad agent startup did not succeed: check logs" + fi + echo -ne "\b\b\b" + done >&2 + + # Create and start the nomad job. + nomad_create_job_file "$dir" + msg "Starting nomad job ..." + # Upon successful job submission, this command will immediately enter + # an interactive monitor. This is useful to watch Nomad's internals make + # scheduling decisions and place the submitted work onto nodes. The + # monitor will end once job placement is done. It is safe to exit the + # monitor early using ctrl+c. + # On successful job submission and scheduling, exit code 0 will be + # returned. If there are job placement issues encountered (unsatisfiable + # constraints, resource exhaustion, etc), then the exit code will be 2. + # Any other errors, including client connection issues or internal + # errors, are indicated by exit code 1. + # FIXME: Timeout for "Deployment "XXX" in progress..." + nomad job run -verbose "$dir/nomad/job-cluster.hcl" + # Assuming that `nomad` placement is enough wait. + local nomad_alloc_id=$(nomad job allocs -json cluster | jq -r '.[0].ID') + setenvjqstr 'nomad_alloc_id' "$nomad_alloc_id" + msg "Nomad job allocation ID is: $nomad_alloc_id" + # Show `--status` of `supervisorctl` inside the container. + local supervisord_url=$(envjqr 'supervisord_url') + local container_supervisor_nix=$( envjqr 'container_supervisor_nix') + local container_supervisord_conf=$(envjqr 'container_supervisord_conf') + msg "Supervisor status inside container ..." + # Print the command used for debugging purposes. + msg "'nomad alloc exec --task node-0 \"$nomad_alloc_id\" \"$container_supervisor_nix\"/bin/supervisorctl --serverurl \"$supervisord_url\" --configuration \"$container_supervisord_conf\" status'" + # Execute the actual command. + nomad alloc exec --task node-0 "$nomad_alloc_id" "$container_supervisor_nix"/bin/supervisorctl --serverurl "$supervisord_url" --configuration "$container_supervisord_conf" status || true + ;; + + describe-run ) + local usage="USAGE: wb nomad $op RUN-DIR" + local dir=${1:?$usage} + + echo " - Nomad job: $(realpath $dir)/nomad/job-cluster.hcl" + ;; + + # Nomad-specific + service-start ) + local usage="USAGE: wb nomad $op RUN-DIR NODE-NAME" + local dir=${1:?$usage}; shift + local service=${1:?$usage}; shift + + backend_nomad nomad-alloc-exec-supervisorctl "$dir" "$service" start "$service" + ;; + + # Nomad-specific + service-stop ) + local usage="USAGE: wb nomad $op RUN-DIR NODE-NAME" + local dir=${1:?$usage}; shift + local service=${1:?$usage}; shift + + backend_nomad nomad-alloc-exec-supervisorctl "$dir" "$service" stop "$service" + ;; + + # Nomad-specific + is-service-running ) + local usage="USAGE: wb nomad $op RUN-DIR DOCKER-SERVICE" + local dir=${1:?$usage}; shift + local service=${1:?$usage}; shift + + backend_nomad nomad-alloc-exec-supervisorctl "$dir" "$service" status "$service" > /dev/null && true + ;; + + # Nomad-specific + nomad-alloc-exec-supervisorctl ) + local usage="USAGE: wb nomad $op RUN-DIR NODE-NAME" + local dir=${1:?$usage}; shift + local task=${1:?$usage}; shift + local action=${1:?$usage}; shift + + local nomad_alloc_id=$(envjqr 'nomad_alloc_id') + local supervisord_url=$(envjqr 'supervisord_url') + local container_supervisor_nix=$(envjqr 'container_supervisor_nix') + local container_supervisord_conf=$(envjqr 'container_supervisord_conf') + nomad alloc exec --task "$task" "$nomad_alloc_id" "$container_supervisor_nix"/bin/supervisorctl --serverurl "$supervisord_url" --configuration "$container_supervisord_conf" "$action" $@ + ;; + + start-node ) + local usage="USAGE: wb nomad $op RUN-DIR NODE-NAME" + local dir=${1:?$usage}; shift + local node=${1:?$usage}; shift + + backend_nomad service-start "$dir" $node + # Always wait for the node to be ready. + backend_nomad wait-node "$dir" $node + ;; + + stop-node ) + local usage="USAGE: wb nomad $op RUN-DIR NODE-NAME" + local dir=${1:?$usage}; shift + local node=${1:?$usage}; shift + + backend_nomad service-stop "$dir" $node + ;; + + wait-node ) + local usage="USAGE: wb nomad $op RUN-DIR [NODE-NAME]" + local dir=${1:?$usage}; shift + local node=${1:-$(dirname $CARDANO_NODE_SOCKET_PATH | xargs basename)}; shift + local socket=$(backend_nomad get-node-socket-path "$dir" $node) + + local patience=$(jq '.analysis.cluster_startup_overhead_s | ceil' $dir/profile.json) i=0 + echo -n "workbench: nomad: waiting ${patience}s for socket of $node: " >&2 + while test ! -S $socket + do printf "%3d" $i; sleep 1 + i=$((i+1)) + if test $i -ge $patience + then echo + progress "nomad" "$(red FATAL): workbench: nomad: patience ran out for $(white $node) after ${patience}s, socket $socket" + backend_nomad stop-cluster "$dir" + fatal "$node startup did not succeed: check logs in $(dirname $socket)/stdout & stderr" + fi + echo -ne "\b\b\b" + done >&2 + echo " $node up (${i}s)" >&2 + ;; + + start-nodes ) + local usage="USAGE: wb nomad $op RUN-DIR [HONOR_AUTOSTART=]" + local dir=${1:?$usage}; shift + local honor_autostart=${1:-} + + local nodes=($(jq_tolist keys "$dir"/node-specs.json)) + for node in ${nodes[*]} + do + if test -n "$honor_autostart" + then + if jqtest ".\"$node\".autostart" "$dir"/node-specs.json + then + backend_nomad start-node "$dir" "$node" + fi + else + backend_nomad start-node "$dir" "$node" + fi + done + + if test ! -v CARDANO_NODE_SOCKET_PATH + then export CARDANO_NODE_SOCKET_PATH=$(backend_nomad get-node-socket-path "$dir" 'node-0') + fi + ;; + + start ) + local usage="USAGE: wb nomad $op RUN-DIR" + local dir=${1:?$usage}; shift + + if jqtest ".node.tracer" "$dir"/profile.json + then + backend_nomad service-start "$dir" tracer + # Wait for tracer socket + # If tracer fails here, the rest of the cluster is brought up without + # any problems. + local socket=$(jq -r '.network.contents' "$dir/tracer/config.json") + local patience=$(jq '.analysis.cluster_startup_overhead_s | ceil' "$dir/profile.json") i=0 + echo -n "workbench: nomad: waiting ${patience}s for socket of tracer: " >&2 + while test ! -S "$dir/tracer/$socket" + do printf "%3d" $i; sleep 1 + i=$((i+1)) + if test $i -ge $patience + then echo + progress "nomad" "$(red FATAL): workbench: nomad: patience ran out for $(white tracer) after ${patience}s, socket $socket" + backend_nomad stop-cluster "$dir" + fatal "$node startup did not succeed: check logs in $(dirname $socket)/stdout & stderr" + fi + echo -ne "\b\b\b" + done >&2 + echo " tracer up (${i}s)" >&2 + fi + ;; + + get-node-socket-path ) + local usage="USAGE: wb nomad $op RUN-DIR NODE-NAME" + local dir=${1:?$usage} + local node_name=${2:?$usage} + + echo -n $dir/$node_name/node.socket + ;; + + start-generator ) + local usage="USAGE: wb nomad $op RUN-DIR" + local dir=${1:?$usage}; shift + + while test $# -gt 0 + do case "$1" in + --* ) msg "FATAL: unknown flag '$1'"; usage_docker;; + * ) break;; esac; shift; done + + backend_nomad service-start "$dir" generator + ;; + + wait-node-stopped ) + local usage="USAGE: wb nomad $op RUN-DIR NODE" + local dir=${1:?$usage}; shift + local node=${1:?$usage}; shift + + progress_ne "docker" "waiting until $node stops: ....." + local i=0 + while backend_nomad is-service-running "$dir" "$node" + do + echo -ne "\b\b\b\b\b"; printf "%5d" $i >&2; i=$((i+1)) + sleep 1 + done >&2 + echo -e "\b\b\b\b\bdone, after $(with_color white $i) seconds" >&2 + ;; + + wait-pools-stopped ) + local usage="USAGE: wb nomad $op RUN-DIR" + local dir=${1:?$usage}; shift + + local i=0 pools=$(jq .composition.n_pool_hosts $dir/profile.json) start_time=$(date +%s) + msg_ne "nomad: waiting until all pool nodes are stopped: 000000" + touch $dir/flag/cluster-termination + + for ((pool_ix=0; pool_ix < $pools; pool_ix++)) + do + while backend_nomad is-service-running "$dir" "node-${pool_ix}" && test -f $dir/flag/cluster-termination + do + echo -ne "\b\b\b\b\b\b"; printf "%6d" $((i + 1)); i=$((i+1)) + sleep 1 + done + echo -ne "\b\b\b\b\b\b"; echo -n "node-${pool_ix} 000000" + done >&2 + echo -ne "\b\b\b\b\b\b" + local elapsed=$(($(date +%s) - start_time)) + if test -f $dir/flag/cluster-termination + then echo " All nodes exited -- after $(yellow $elapsed)s" >&2 + else echo " Termination requested -- after $(yellow $elapsed)s" >&2; fi + ;; + + stop-cluster ) + local usage="USAGE: wb nomad $op RUN-DIR" + local dir=${1:?$usage}; shift + + backend_nomad nomad-alloc-exec-supervisorctl "$dir" generator stop all + backend_nomad nomad-alloc-exec-supervisorctl "$dir" tracer stop all + for node in $(jq_tolist 'keys' "$dir"/node-specs.json) + do + backend_nomad nomad-alloc-exec-supervisorctl "$dir" "$node" stop all + done + + nomad job stop -global -no-shutdown-delay -purge -yes cluster + local nomad_pid=$(envjqr 'nomad_pid') + kill -SIGINT "$nomad_pid" + ;; + + cleanup-cluster ) + local usage="USAGE: wb nomad $op RUN-DIR" + local dir=${1:?$usage}; shift + + msg "nomad: resetting cluster state in: $dir" + rm -f $dir/*/std{out,err} $dir/node-*/*.socket $dir/*/logs/* 2>/dev/null || true + rm -fr $dir/node-*/state-cluster/ + # Clean nomad logs. + rm -f $dir/nomad/nomad.log $dir/nomad/std{out,err} + rm -rf $dir/nomad/data/* + ;; + + * ) usage_docker;; esac +} + +# Configure `nomad` and its `podman` plugin / task driver +# (Task Drivers are also called plugins because they are pluggable). +# +# WARNING: `podman`/`skopeo` are run using default parameters. Every workbench +# user is responsible for its local/global configurations. +# TODO: Unless this breaks reproducibility and with every call config files +# and parameters need to be overriden. +# For example: +# Local version of /etc/containers/containers.conf +# mkdir -p $HOME/.config/containers/ +# touch $HOME/.config/containers/containers.conf +# CONTAINERS_CONF=$HOME/.config/containers/containers.conf +# Local version of /etc/containers/storage.conf +# https://www.mankier.com/5/containers-storage.conf +# mkdir -p $HOME/.local/share/containers/storage/volumes +# touch $HOME/.config/containers/storage.conf +# CONTAINERS_STORAGE_CONF=$HOME/.config/containers/storage.conf +# Local version of /etc/containers/policy.json +# https://www.mankier.com/5/containers-policy.json +# mkdir -p $HOME/.config/containers/ +# touch $HOME/.config/containers/policy.json +nomad_create_folders_and_config() { + local dir=$1 + # Folders: + mkdir -p "$dir/nomad/config" + mkdir -p "$dir/nomad/data" + mkdir -p "$dir/nomad/data/plugins" + # Podman Task Driver - Client Requirements: + # "Ensure that Nomad can find the plugin, refer to `plugin_dir`." + # https://www.nomadproject.io/plugins/drivers/podman#client-requirements + ln -s "$(which nomad-driver-podman)" "$dir/nomad/data/plugins/nomad-driver-podman" + # Config: + # - `nomad` configuration docs: + # - - https://www.nomadproject.io/docs/configuration + # - Generic `nomad` plugins / task drivers configuration docs: + # - - https://www.nomadproject.io/plugins/drivers + # - - https://www.nomadproject.io/docs/configuration/plugin + # - Specific `nomad` `podman` plugin / task driver configuration docs: + # - - https://www.nomadproject.io/plugins/drivers/podman#plugin-options + # - - https://github.com/hashicorp/nomad-driver-podman#driver-configuration + cat > "$dir/nomad/config/nomad.hcl" <<- EOF + region = "workbench" + datacenter = "workbench" + name = "workbench" + data_dir = "$dir/nomad/data" + plugin_dir = "$dir/nomad/data/plugins" + bind_addr = "127.0.0.1" + ports = { + http = 4646 + } + log_level = "INFO" + log_json = true + log_file = "$dir/nomad/" + leave_on_interrupt = true + leave_on_terminate = true + plugin "nomad-driver-podman" { + args = [] + config { + # TODO: Use custom socket location! + # socket_path = "unix:$dir/nomad/podman.sock" + volumes { + enabled = true + } + recover_stopped = false + gc { + container = false + } + } + } +EOF +} + +# Start the `podman` API service needed by `nomad`. +nomad_start_podman_service() { + local dir=$1 + # TODO: Use custom socket location! + # podman --url "unix:$dir/nomad/podman.sock" system service --time 60 "unix:$dir/nomad/podman.sock" & + local socket="/run/user/$UID/podman/podman.sock" +# if test -S "$socket" +# then +# msg "Podman API service was already running" +# else + # The session is kept open waiting for a new connection for 60 seconds. + # https://discuss.hashicorp.com/t/nomad-podman-rhel8-driver-difficulties/21877/4 + # `--time`: Time until the service session expires in seconds. Use 0 + # to disable the timeout (default 5). + podman system service --time 60 & + local i=0 + local patience=5 + while test ! -S "$socket" + do printf "%3d" $i; sleep 1 + i=$((i+1)) + if test $i -ge $patience + then echo + progress "nomad-driver-podman" "$(red FATAL): workbench: nomad-driver-podman: patience ran out after ${patience}s, socket $socket" + backend_nomad stop-cluster "$dir" + fatal "nomad-driver-podman startup did not succeed: check logs" + fi + echo -ne "\b\b\b" + done >&2 +# fi + msg "Podman API service started" +} + +# Need to use HCL instead of JSON. The only workaround is to send commands to +# Nomad using `curl` instead of the command line (`nomad job ...`). +# - "The nomad job run command currently accepts only HCL" +# [https://github.com/hashicorp/nomad/issues/6758#issuecomment-794116722] +nomad_create_job_file() { + local dir=$1 + local container_mountpoint=$( envjqr 'container_mountpoint') + # If CARDANO_MAINNET_MIRROR is present attach it as a volume. + if test -n "$CARDANO_MAINNET_MIRROR" + then + # The nix-store path contains 3 levels of symlinks. This is a hack to + # avoid creating a container image with all these files. + local immutable_store=$(readlink -f "$CARDANO_MAINNET_MIRROR"/immutable) + local optional_volumes="[ + \"$CARDANO_MAINNET_MIRROR:$CARDANO_MAINNET_MIRROR:ro\" + , \"$immutable_store:$immutable_store:ro\" + $(find -L "$immutable_store" -type f -exec realpath {} \; | xargs dirname | sort | uniq | xargs -I "{}" echo ", \"{}:{}:ro\"") + ]" + else + local optional_volumes="[]" + fi + # Volumes + local jq_filter=" + [ + \"${dir}:/tmp/cluster/run/current:rw,exec\" + ] + + + ( . | keys | map( \"${dir}/genesis:${container_mountpoint}/\" + . + \"/genesis:ro\" ) ) + + + ( . | keys | map( \"${dir}/\" + . + \":${container_mountpoint}/generator/\" + . + \":ro\" ) ) + + + ( . | keys | map( \"${dir}/genesis:${container_mountpoint}/generator/\" + . + \"/genesis:ro\" ) ) + + + [ + \"${dir}/genesis:${container_mountpoint}/generator/genesis:ro\" + , \"${dir}/genesis/utxo-keys:${container_mountpoint}/generator/genesis/utxo-keys:ro\" + ] + + + \$optional_volumes + " + local podman_volumes=$(jq "$jq_filter" --argjson optional_volumes "$optional_volumes" "$dir"/profile/node-specs.json) + # Create the task to run in `nomad` using `podman` driver. + # https://www.nomadproject.io/docs/job-specification + # https://www.nomadproject.io/docs/job-specification/job + # https://github.com/hashicorp/nomad-driver-podman#task-configuration +cat > "$dir/nomad/job-cluster.hcl" <<- EOF +job "cluster" { + region = "workbench" + datacenters = [ "workbench" ] + type = "service" + reschedule { + attempts = 0 + unlimited = false + } + # A group defines a series of tasks that should be co-located + # on the same client (host). All tasks within a group will be + # placed on the same host. + group "cluster" { + restart { + attempts = 0 + mode = "fail" + } + # The network stanza specifies the networking requirements for the task + # group, including the network mode and port allocations. + # https://developer.hashicorp.com/nomad/docs/job-specification/network + network { + mode = "host" + } +EOF + # Cluster +# local task_stanza_name_c="cluster" +# local task_stanza_file_c="$dir/nomad/job-cluster-task-$task_stanza_name_c.hcl" +# nomad_create_task_stanza "$task_stanza_file_c" "$task_stanza_name_c" "$podman_volumes" +#cat "$task_stanza_file_c" >> "$dir/nomad/job-cluster.hcl" + # Nodes + for node in $(jq_tolist 'keys' "$dir"/node-specs.json) + do + local task_stanza_name="$node" + local task_stanza_file="$dir/nomad/job-cluster-task-$task_stanza_name.hcl" + nomad_create_task_stanza "$task_stanza_file" "$task_stanza_name" "$podman_volumes" +cat "$task_stanza_file" >> "$dir/nomad/job-cluster.hcl" + done + # Tracer + local task_stanza_name_t="tracer" + local task_stanza_file_t="$dir/nomad/job-cluster-task-$task_stanza_name_t.hcl" + nomad_create_task_stanza "$task_stanza_file_t" "$task_stanza_name_t" "$podman_volumes" +cat "$task_stanza_file_t" >> "$dir/nomad/job-cluster.hcl" + # Generator + local task_stanza_name_g="generator" + local task_stanza_file_g="$dir/nomad/job-cluster-task-$task_stanza_name_g.hcl" + nomad_create_task_stanza "$task_stanza_file_g" "$task_stanza_name_g" "$podman_volumes" +cat "$task_stanza_file_g" >> "$dir/nomad/job-cluster.hcl" + # The end. +cat >> "$dir/nomad/job-cluster.hcl" <<- EOF + } +} +EOF +} + +nomad_create_task_stanza() { + local file=$1 + local name=$2 + local podman_volumes=$3 + local oci_image_name=$( envjqr 'oci_image_name') + local oci_image_tag=$( envjqr 'oci_image_tag') + local container_workdir=$( envjqr 'container_workdir') + local container_supervisor_nix=$( envjqr 'container_supervisor_nix') + local container_supervisord_conf=$(envjqr 'container_supervisord_conf') + cat > "$file" <<- EOF +# The task stanza creates an individual unit of work, such as a +# Docker container, web application, or batch processing. +task "$name" { + driver = "podman" + config { + image = "${oci_image_name}:${oci_image_tag}" + force_pull = false + # TODO/FIXME: Don't know how to make podman log to nomad + # instead of journald. + # No argument or block type is named \"logging\" + #logging = { + # driver = "nomad" + #} + hostname = "$name" + network_mode = "host" + tmpfs = [ + "/tmp" + ] + volumes = ${podman_volumes} + working_dir = "${container_workdir}" + } + env = { + SUPERVISOR_NIX = "${container_supervisor_nix}" + SUPERVISORD_CONFIG = "${container_supervisord_conf}" + } + # Avoid: podman WARN[0066] StopSignal SIGTERM failed to stop container + # cluster-XX in 5 seconds, resorting to SIGKILL + kill_timeout = 15 +} +EOF +} diff --git a/nix/workbench/backend/supervisor-conf.nix b/nix/workbench/backend/supervisor-conf.nix index 7ad0b53dec4..9b03b38880e 100644 --- a/nix/workbench/backend/supervisor-conf.nix +++ b/nix/workbench/backend/supervisor-conf.nix @@ -3,6 +3,8 @@ , stateDir , basePort , node-services +, unixHttpServerPort ? null +, inetHttpServerPort ? null ## Last-moment overrides: , extraBackendConfig }: @@ -23,14 +25,24 @@ let strip_ansi = true; }; supervisorctl = {}; - inet_http_server = { - port = "127.0.0.1:9001"; - }; "rpcinterface:supervisor" = { "supervisor.rpcinterface_factory" = "supervisor.rpcinterface:make_main_rpcinterface"; }; } // + lib.attrsets.optionalAttrs (unixHttpServerPort != null) { + unix_http_server = { + file = unixHttpServerPort; + chmod = "0777"; + }; + } + // + lib.attrsets.optionalAttrs (inetHttpServerPort != null) { + inet_http_server = { + port = inetHttpServerPort; + }; + } + // listToAttrs (mapAttrsToList (_: nodeSvcSupervisorProgram) node-services) // diff --git a/nix/workbench/backend/supervisor-run.nix b/nix/workbench/backend/supervisor-run.nix index f1b4f890e2f..837d2998b76 100644 --- a/nix/workbench/backend/supervisor-run.nix +++ b/nix/workbench/backend/supervisor-run.nix @@ -143,7 +143,9 @@ in in { inherit stateDir; + inherit profileName; inherit workbench supervisord-workbench; + inherit (supervisord-workbench) backend; inherit profileNix profile topology genesis; inherit interactive-start interactive-stop interactive-restart; inherit profile-run; diff --git a/nix/workbench/backend/supervisor.nix b/nix/workbench/backend/supervisor.nix index 40740bd3515..44f0d23ea60 100644 --- a/nix/workbench/backend/supervisor.nix +++ b/nix/workbench/backend/supervisor.nix @@ -14,14 +14,34 @@ in ## , ... }: -with lib; let backend = rec { name = "supervisor"; + # Unlike the nomad backend `useCabalRun` is honored here. + inherit useCabalRun; + services-config = import ./services-config.nix {inherit lib workbench basePort stateDir useCabalRun enableEKG;}; + extraShellPkgs = with pkgs; [ + python3Packages.supervisor + ] + ++ lib.optionals ( useCabalRun) + (with haskellPackages; [ + cabalWrapped + ghcid + haskellBuildUtils + cabal-plan + ]) + ## Workbench's main script is called directly in dev mode. + ++ lib.optionals (!useCabalRun) + (with cardanoNodePackages; [ + cardano-node + cardano-tracer + tx-generator + ]); + materialise-profile = { profileNix }: pkgs.runCommand "workbench-backend-output-${profileNix.name}-${name}d" @@ -35,6 +55,7 @@ let pkgs lib stateDir basePort extraBackendConfig; + inetHttpServerPort = "127.0.0.1:9001"; }; } '' diff --git a/nix/workbench/default.nix b/nix/workbench/default.nix index 5a45ae38c23..62cbbcf0f1c 100644 --- a/nix/workbench/default.nix +++ b/nix/workbench/default.nix @@ -40,12 +40,11 @@ let jq moreutils procps - cardano-cli cardano-topology - ] ++ lib.optional (!pkgs.stdenv.hostPlatform.isDarwin) db-analyser ++ [ - locli - ]); + ] ++ lib.optional (!pkgs.stdenv.hostPlatform.isDarwin) db-analyser + ++ [ locli ] + ); runWorkbench = name: command: diff --git a/nix/workbench/run.sh b/nix/workbench/run.sh index 5e82294a3f4..1ff684a17d9 100644 --- a/nix/workbench/run.sh +++ b/nix/workbench/run.sh @@ -370,6 +370,7 @@ EOF local usage="USAGE: wb run $op BATCH-NAME PROFILE-NAME [ENV-CONFIG-OPTS..] [-- BACKEND-ARGS-AND-ENV-CONFIG-OPTS..]" local batch=${1:?$usage}; shift local profile_name=${1:?$usage}; shift + local backend_name=${1:?$usage}; shift local profile= topology= genesis_cache_entry= manifest= preset= cabal_mode= while test $# -gt 0 @@ -421,7 +422,18 @@ EOF profile describe-timing "$timing" ## 3. decide the tag: - local run=$(jq '.start_tag' -r <<<$timing)$(if test "$batch" != 'plain'; then echo -n .$batch; fi).$hash.$profile_name$(test -z "$cabal_mode" && echo '.nix') + if [ "$backend_name" == "supervisor" ]; + then + local backend_identifier="sup" + else + if [ "$backend_name" == "nomad" ]; + then + local backend_identifier="nom" + else + local backend_identifier="$backend_name" + fi + fi + local run=$(jq '.start_tag' -r <<<$timing)$(if test "$batch" != 'plain'; then echo -n .$batch; fi).$hash.$profile_name.$backend_identifier progress "run | tag" "allocated run identifier (tag): $(with_color white $run)" ## 4. allocate directory: diff --git a/nix/workbench/shell.nix b/nix/workbench/shell.nix index 7c4893edf5b..462e4474b95 100644 --- a/nix/workbench/shell.nix +++ b/nix/workbench/shell.nix @@ -7,9 +7,8 @@ ## , cardano-mainnet-mirror ## -, profileName +, workbenchRun , workbenchDevMode ? false -, useCabalRun ? false ## , profiled ? false , withHoogle ? true @@ -18,18 +17,15 @@ with lib; -let cluster = pkgs.supervisord-workbench-for-profile { - inherit profileName useCabalRun profiled; - }; - - inherit (cluster) profile; +let + inherit (workbenchRun) profileName backend profile; - shellHook = { workbenchDevMode, useCabalRun, profiled, profileName, withMainnet }: '' + shellHook = { profileName, backend, workbenchDevMode, profiled, withMainnet }: '' while test $# -gt 0 do shift; done ## Flush argv[] - echo 'workbench shellHook: workbenchDevMode=${toString workbenchDevMode} useCabalRun=${toString useCabalRun} profiled=${toString profiled} profileName=${profileName}' - export WB_BACKEND=supervisor + echo 'workbench shellHook: profileName=${profileName} backendName=${backend.name} useCabalRun=${toString backend.useCabalRun} workbenchDevMode=${toString workbenchDevMode} profiled=${toString profiled} ' + export WB_BACKEND=${backend.name} export WB_SHELL_PROFILE=${profileName} export WB_SHELL_PROFILE_DIR=${profile} @@ -46,7 +42,7 @@ let cluster = pkgs.supervisord-workbench-for-profile { ''} ${optionalString - useCabalRun + backend.useCabalRun '' . nix/workbench/lib.sh . nix/workbench/lib-cabal.sh ${optionalString profiled "--profiled"} @@ -74,7 +70,7 @@ let cluster = pkgs.supervisord-workbench-for-profile { in project.shellFor { name = "workbench-shell"; - shellHook = shellHook { inherit workbenchDevMode useCabalRun profiled profileName withMainnet; }; + shellHook = shellHook { inherit profileName backend workbenchDevMode profiled withMainnet; }; inherit withHoogle; @@ -97,19 +93,14 @@ in project.shellFor { # These programs will be available inside the nix-shell. nativeBuildInputs = with pkgs; with haskellPackages; with cardanoNodePackages; [ cardano-ping - cabalWrapped db-analyser - ghcid - haskellBuildUtils pkgs.graphviz graphmod - cabal-plan weeder nixWrapped pkgconfig profiteur profiterole - python3Packages.supervisor ghc-prof-flamegraph sqlite-interactive tmux @@ -118,24 +109,17 @@ in project.shellFor { pkgs.moreutils pkgs.pstree pkgs.time - cluster.interactive-start - cluster.interactive-stop - cluster.interactive-restart + workbenchRun.interactive-start + workbenchRun.interactive-stop + workbenchRun.interactive-restart ] ++ lib.optional haveGlibcLocales pkgs.glibcLocales - ## Workbench's main script is called directly in dev mode. - ++ lib.optionals (!useCabalRun) - [ - cardano-cli - cardano-node - cardano-topology - cardano-tracer - locli - tx-generator - ] + ++ lib.optionals (!backend.useCabalRun) [cardano-topology cardano-cli locli] + ++ backend.extraShellPkgs ++ lib.optionals (!workbenchDevMode) [ - cluster.workbench.workbench - ]; + workbenchRun.workbench.workbench + ] + ; } // { inherit shellHook; } diff --git a/nix/workbench/wb b/nix/workbench/wb index 429a6e43c8c..1bf94bbfe84 100755 --- a/nix/workbench/wb +++ b/nix/workbench/wb @@ -20,6 +20,7 @@ global_basedir=${global_basedir:-$(realpath "$(dirname "$0")")} . "$global_basedir"/explain-mode.sh . "$global_basedir"/backend.sh +. "$global_basedir"/backend/nomad.sh . "$global_basedir"/backend/supervisor.sh usage_main() { @@ -128,7 +129,7 @@ start() { local batch_name= local profile_name= profile= - local backend=supervisor + local backend=${WB_BACKEND:-supervisor} local node_source=. local node_rev= local cabal_mode= @@ -212,7 +213,7 @@ start() ${run_allocate_args[@]} --manifest "$manifest" ) - run ${run_args[@]} 'allocate' $batch_name $profile_name "${args[@]}" + run ${run_args[@]} 'allocate' $batch_name $profile_name $backend "${args[@]}" local run=$(run current-tag) current_run_path=$(run current-path) diff --git a/shell.nix b/shell.nix index c56184ba6a9..2386ae8eb51 100644 --- a/shell.nix +++ b/shell.nix @@ -4,6 +4,7 @@ let defaultCustomConfig = import ./nix/custom-config.nix defaultCustomConfig; in { withHoogle ? defaultCustomConfig.withHoogle , profileName ? defaultCustomConfig.localCluster.profileName +, backendName ? defaultCustomConfig.localCluster.backendName , workbenchDevMode ? defaultCustomConfig.localCluster.workbenchDevMode , useCabalRun ? true , customConfig ? { @@ -55,26 +56,46 @@ let haveGlibcLocales = pkgs.glibcLocales != null && stdenv.hostPlatform.libc == "glibc"; - workbench-shell = with customConfig.localCluster; - import ./nix/workbench/shell.nix - { inherit pkgs lib haskellLib project; - inherit setLocale haveGlibcLocales commandHelp; - inherit cardano-mainnet-mirror; - inherit profileName workbenchDevMode useCabalRun; - inherit profiled withHoogle; - }; + workbench-shell = + let + workbenchRun = + if backendName == "nomad" + then pkgs.nomad-workbench-for-profile + { inherit profileName useCabalRun profiled; } + # Supervidor by default. + else pkgs.supervisord-workbench-for-profile + { inherit profileName useCabalRun profiled; } + ; + in with customConfig.localCluster; + import ./nix/workbench/shell.nix + { inherit pkgs lib haskellLib project; + inherit setLocale haveGlibcLocales commandHelp; + inherit cardano-mainnet-mirror; + inherit workbenchRun workbenchDevMode; + inherit profiled withHoogle; + }; devops = - let devopsShellParams = - { inherit workbenchDevMode profiled; - profileName = "devops-bage"; - withMainnet = false; + let profileName = "devops-bage"; + workbenchRun = pkgs.supervisord-workbench-for-profile + { + inherit profileName; useCabalRun = false; }; - cluster = pkgs.supervisord-workbench-for-profile - { - inherit (devopsShellParams) profileName useCabalRun; + devopsShellParams = + { inherit profileName; + backend = workbenchRun.backend; + inherit workbenchDevMode profiled; + withMainnet = false; }; + devopsShell = with customConfig.localCluster; + import ./nix/workbench/shell.nix + { inherit pkgs lib haskellLib project; + inherit setLocale haveGlibcLocales commandHelp; + inherit cardano-mainnet-mirror; + inherit workbenchRun workbenchDevMode; + inherit profiled withHoogle; + }; in project.shellFor { name = "devops-shell"; @@ -94,11 +115,11 @@ let pkgs.graphviz python3Packages.supervisor python3Packages.ipython - cluster.interactive-start - cluster.interactive-stop - cluster.interactive-restart + workbenchRun.interactive-start + workbenchRun.interactive-stop + workbenchRun.interactive-restart cardanolib-py - cluster.workbench.workbench + workbenchRun.workbench.workbench pstree pkgs.time ]; @@ -108,10 +129,10 @@ let | ${figlet}/bin/figlet -f banner -c \ | ${lolcat}/bin/lolcat - ${workbench-shell.shellHook devopsShellParams} + ${devopsShell.shellHook devopsShellParams} # Socket path default to first node launched by "start-cluster": - export CARDANO_NODE_SOCKET_PATH=$(wb backend get-node-socket-path ${cluster.stateDir} 'node-0') + export CARDANO_NODE_SOCKET_PATH=$(wb backend get-node-socket-path ${workbenchRun.stateDir} 'node-0') ${setLocale}