diff --git a/.github/workflows/test-import-using-nix.yml b/.github/workflows/test-import-using-nix.yml new file mode 100644 index 000000000..09e57dc05 --- /dev/null +++ b/.github/workflows/test-import-using-nix.yml @@ -0,0 +1,29 @@ +on: + workflow_dispatch: # allow manual execution + push: + pull_request: + schedule: + # run on the 3rd each month at 10:00am + - cron: '0 10 3 * *' + +jobs: + nix-check-and-import: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v11 + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + with: + nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/20.09.tar.gz + install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-3.0pre20201007_5257a25/install + extra_nix_config: | + experimental-features = nix-command flakes + - name: run checks & test import + run: | + cd etc/nix + nix --print-build-logs flake check + ./test-import-using-nix.sh alpine diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 634718bbf..ba414b5e9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,14 @@ Release notes -------------- -### Version v20.10 +============= + +vNext +----- + +- Add support for nix +- Improve documentation + + +Version v20.10 +-------------- This release comes with the new calver versioning scheme and an initial data dump. diff --git a/README.rst b/README.rst index 831dca24b..f974b9810 100644 --- a/README.rst +++ b/README.rst @@ -161,6 +161,44 @@ for this purpose:: SECRET_KEY=$(python -c "from django.core.management import utils; print(utils.get_random_secret_key())") +Using Nix +~~~~~~~~~ + +You can install VulnerableCode with `Nix `__ +(`Flake `__ support is needed):: + + cd etc/nix + nix --print-build-logs flake check # build & run tests + +There are several options to use the Nix version:: + + # Enter an interactive environment with all dependencies set up. + cd etc/nix + nix develop + > ../../manage.py ... # invoke the local checkout + > vulnerablecode-manage.py ... # invoke manage.py as installed in the nix store + + # Test the import prodecure using the Nix version. + etc/nix/test-import-using-nix.sh --all # import everything + # Test the import using the local checkout. + INSTALL_DIR=. etc/nix/test-import-using-nix.sh ruby # import ruby only + + +**Keeping the Nix setup in sync** + +The Nix installation uses `mach-nix `__ to +handle Python dependencies because some dependencies are currently not available +as Nix packages. All Python dependencies are automatically fetched from +``./requirements.txt``. If the ``mach-nix``-based installation fails, you might +need to update ``mach-nix`` itself and the `pypi-deps-db +`_ version in use (see +``etc/nix/flake.nix:inputs.machnix`` and ``machnixFor.pypiDataRev``). + +Non-Python dependencies are curated in +``etc/nix/flake.nix:vulnerablecode.propagatedBuildInputs``. + + + Run Tests --------- diff --git a/etc/nix/default.nix b/etc/nix/default.nix new file mode 100644 index 000000000..b3f68195c --- /dev/null +++ b/etc/nix/default.nix @@ -0,0 +1,4 @@ +(import (fetchTarball + "https://github.com/edolstra/flake-compat/archive/master.tar.gz") { + src = ./.; + }).defaultNix diff --git a/etc/nix/flake.lock b/etc/nix/flake.lock new file mode 100644 index 000000000..a91de1df5 --- /dev/null +++ b/etc/nix/flake.lock @@ -0,0 +1,96 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1601282935, + "narHash": "sha256-WQAFV6sGGQxrRs3a+/Yj9xUYvhTpukQJIcMbIi7LCJ4=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "588973065fce51f4763287f0fda87a174d78bf48", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "machnix": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "pypi-deps-db": "pypi-deps-db" + }, + "locked": { + "lastModified": 1606460537, + "narHash": "sha256-gcKjBA1fpCzh0nGoaxurIjIJanPDrAea6NK60M+vfRk=", + "owner": "DavHau", + "repo": "mach-nix", + "rev": "1ec92303acd142aa1a3b60bb97745544cf049312", + "type": "github" + }, + "original": { + "owner": "DavHau", + "ref": "3.1.1", + "repo": "mach-nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1605988311, + "narHash": "sha256-PA+kgq46NApOAJlmBNJHs5DwsIrY+jodM0e4g7VtXyY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "2247d824fe07f16325596acc7faa286502faffd1", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1603791972, + "narHash": "sha256-nj2SvACFH+NERpye1kudcuygCcvnsZYv26M2+wgM5vE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "cd63096d6d887d689543a0b97743d28995bc9bc3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "20.09", + "repo": "nixpkgs", + "type": "github" + } + }, + "pypi-deps-db": { + "flake": false, + "locked": { + "lastModified": 1606280641, + "narHash": "sha256-sVYIBMtvZgxQQkbxZg/45xw4e7vVh+5SU1kkvUvtnrc=", + "owner": "DavHau", + "repo": "pypi-deps-db", + "rev": "63bb8887c8de8e056f5ed77ac6a35c771c0c5d57", + "type": "github" + }, + "original": { + "owner": "DavHau", + "repo": "pypi-deps-db", + "type": "github" + } + }, + "root": { + "inputs": { + "machnix": "machnix", + "nixpkgs": "nixpkgs_2" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/etc/nix/flake.nix b/etc/nix/flake.nix new file mode 100644 index 000000000..0ae9f6627 --- /dev/null +++ b/etc/nix/flake.nix @@ -0,0 +1,153 @@ +{ + description = + "Vulnerablecode - A free and open vulnerabilities database and the packages they impact."; + + inputs.nixpkgs = { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + ref = "20.09"; + }; + + inputs.machnix = { + type = "github"; + owner = "DavHau"; + repo = "mach-nix"; + ref = "3.1.1"; + }; + + outputs = { self, nixpkgs, machnix }: + let + + vulnerablecode-src = ./../..; + + # Extract version from setup.py. + version = builtins.head (builtins.match ''.*version=["']?([^"',]+).*'' + (builtins.readFile (vulnerablecode-src + "/setup.py"))); + + # Common shell code. + libSh = ./lib.sh; + + # System types to support. + supportedSystems = [ "x86_64-linux" ]; + + # Helper function to generate an attrset '{ x86_64-linux = f "x86_64-linux"; ... }'. + forAllSystems = f: + nixpkgs.lib.genAttrs supportedSystems (system: f system); + + # Nixpkgs instantiated for supported system types. + nixpkgsFor = forAllSystems (system: + import nixpkgs { + inherit system; + overlays = [ self.overlay ]; + }); + + # mach-nix instantiated for supported system types. + machnixFor = forAllSystems (system: + import machnix { + pkgs = (nixpkgsFor.${system}).pkgs; + python = "python38"; + + # Pin pypi repo to a specific commit which includes all necessary + # Python deps. The default version is updated with every mach-nix + # release might be be sufficient for newer releases. + # The corresponding sha256 hash can be obtained with: + # $ nix-prefetch-url --unpack https://github.com/DavHau/pypi-deps-db/tarball/ + pypiDataRev = "c86b4490a7d838bd54a2d82730455e96c6e4eb14"; + pypiDataSha256 = + "0al490gi0qda1nkb9289z2msgpc633rv5hn3w5qihkl1rh88dmjd"; + }); + + in { + + # A Nixpkgs overlay. + overlay = final: prev: + with final.pkgs; { + + pythonEnv = machnixFor.${system}.mkPython { + requirements = + builtins.readFile (vulnerablecode-src + "/requirements.txt"); + }; + + vulnerablecode = stdenv.mkDerivation { + inherit version; + name = "vulnerablecode-${version}"; + src = vulnerablecode-src; + dontConfigure = true; # do not use ./configure + propagatedBuildInputs = [ pythonEnv postgresql ]; + + postPatch = '' + # Make sure the pycodestyle binary in $PATH is used. + substituteInPlace vulnerabilities/tests/test_basics.py \ + --replace 'join(bin_dir, "pycodestyle")' '"pycodestyle"' + ''; + + installPhase = '' + cp -r . $out + ''; + }; + + }; + + # Provide a nix-shell env to work with vulnerablecode. + devShell = forAllSystems (system: + with nixpkgsFor.${system}; + mkShell { + # will be available as env var in `nix develop` / `nix-shell`. + VULNERABLECODE_INSTALL_DIR = vulnerablecode; + buildInputs = [ vulnerablecode ]; + shellHook = '' + alias vulnerablecode-manage.py=${vulnerablecode}/manage.py + ''; + }); + + # Provide some packages for selected system types. + packages = forAllSystems + (system: { inherit (nixpkgsFor.${system}) vulnerablecode; }); + + # The default package for 'nix build'. + defaultPackage = + forAllSystems (system: self.packages.${system}.vulnerablecode); + + # Tests run by 'nix flake check' and by Hydra. + checks = forAllSystems (system: { + inherit (self.packages.${system}) vulnerablecode; + + vulnerablecode-test = with nixpkgsFor.${system}; + stdenv.mkDerivation { + name = "${vulnerablecode.name}-test"; + + buildInputs = [ wget vulnerablecode ]; + + # Used by pygit2. + # See https://github.com/NixOS/nixpkgs/pull/72544#issuecomment-582674047. + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + + unpackPhase = "true"; + + buildPhase = '' + source ${libSh} + initPostgres $(pwd) + export DJANGO_DEV=1 + ${vulnerablecode}/manage.py migrate + ''; + + doCheck = true; + checkPhase = '' + # Run pytest on the installed version. A running postgres + # database server is needed. + (cd ${vulnerablecode} && pytest) + + # Launch the webserver and call the API. + ${vulnerablecode}/manage.py runserver & + sleep 2 + wget http://127.0.0.1:8000/api/ + kill %1 # kill background task (i.e. webserver) + ''; + + installPhase = + "mkdir -p $out"; # make this derivation return success + }; + }); + }; +} diff --git a/etc/nix/lib.sh b/etc/nix/lib.sh new file mode 100644 index 000000000..fc75cf5c9 --- /dev/null +++ b/etc/nix/lib.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Setup postgres; see the README for the latest instructions. +# +# $RUNDIR is used to prevent postgres from accessings its default run dir at +# /run/postgresql. See +# https://github.com/NixOS/nixpkgs/issues/83770#issuecomment-607992517 +function initPostgres() { + ROOTDIR=$1 + DATADIR=$ROOTDIR/pgdata + RUNDIR=$ROOTDIR/run + ENCODING="UTF-8" + mkdir -p "$RUNDIR" + initdb -D "$DATADIR" -E $ENCODING + pg_ctl -D "$DATADIR" -o "-k $RUNDIR" -l "$DATADIR/logfile" start + createuser --host "$RUNDIR" --no-createrole --no-superuser --login --inherit --createdb vulnerablecode + createdb --host "$RUNDIR" -E $ENCODING --owner=vulnerablecode --user=vulnerablecode --port=5432 vulnerablecode +} diff --git a/etc/nix/shell.nix b/etc/nix/shell.nix new file mode 100644 index 000000000..330df0ab6 --- /dev/null +++ b/etc/nix/shell.nix @@ -0,0 +1,3 @@ +(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) { + src = ./.; +}).shellNix diff --git a/etc/nix/test-import-using-nix.sh b/etc/nix/test-import-using-nix.sh new file mode 100755 index 000000000..641289ecb --- /dev/null +++ b/etc/nix/test-import-using-nix.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash + +# Populate a test database using either the Nix installation or the local +# checkout. + +set -e + +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +DEFAULT_INSTALL_DIR=$VULNERABLECODE_INSTALL_DIR # in the Nix store, see flake.nix +INSTALL_DIR=${INSTALL_DIR:-$DEFAULT_INSTALL_DIR} +ARGS=$(if [ $# -eq 0 ]; then echo "--all"; else echo "$@"; fi) +export DJANGO_DEV=${DJANGO_DEV:-1} +TEMPDIR=$(mktemp -d -p "$THIS_DIR") +export TEMPDIR + +source "$THIS_DIR/lib.sh" + +cleanup() { + pg_ctl -D "$DATADIR" stop + rm -rf "$TEMPDIR" +} + +trap cleanup EXIT + +initPostgres "$TEMPDIR" + +"$INSTALL_DIR/manage.py" migrate +"$INSTALL_DIR/manage.py" import $ARGS