Merge remote latest main branch

Signed-off-by: Philippe Ombredanne <[email protected]>

.github/workflows/test-import-using-nix.yml

@@ -0,0 +1,29 @@
+on:
+  workflow_dispatch: # allow manual execution
+  push:
+  pull_request:
+  schedule:
+    # run on the 3rd each month at 10:00am
+    - cron: '0 10 3 * *'
+
+jobs:
+  nix-check-and-import:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: cachix/install-nix-action@v11
+        env:
+          ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+        with:
+          nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/20.09.tar.gz
+          install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-3.0pre20201007_5257a25/install
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+      - name: run checks & test import
+        run: |
+          cd etc/nix
+          nix --print-build-logs flake check
+          ./test-import-using-nix.sh alpine

CHANGELOG.rst

@@ -1,5 +1,14 @@
 Release notes
--------------
-### Version v20.10
+=============
+
+vNext
+-----
+
+- Add support for nix
+- Improve documentation
+
+
+Version v20.10
+--------------
 
 This release comes with the new calver versioning scheme and an initial data dump.

README.rst

@@ -161,6 +161,44 @@ for this purpose::
     SECRET_KEY=$(python -c "from django.core.management import utils; print(utils.get_random_secret_key())")
 
 
+Using Nix
+~~~~~~~~~
+
+You can install VulnerableCode with `Nix <https://nixos.org/download.html>`__ 
+(`Flake <https://nixos.wiki/wiki/Flakes>`__ support is needed)::
+
+    cd etc/nix
+    nix --print-build-logs flake check # build & run tests
+
+There are several options to use the Nix version::
+
+    # Enter an interactive environment with all dependencies set up.
+    cd etc/nix
+    nix develop
+    > ../../manage.py ... # invoke the local checkout
+    > vulnerablecode-manage.py ... # invoke manage.py as installed in the nix store
+
+    # Test the import prodecure using the Nix version.
+    etc/nix/test-import-using-nix.sh --all # import everything
+    # Test the import using the local checkout.
+    INSTALL_DIR=. etc/nix/test-import-using-nix.sh ruby # import ruby only
+
+
+**Keeping the Nix setup in sync**
+
+The Nix installation uses `mach-nix <https://github.com/DavHau/mach-nix>`__ to
+handle Python dependencies because some dependencies are currently not available
+as Nix packages. All Python dependencies are automatically fetched from
+``./requirements.txt``. If the ``mach-nix``-based installation fails, you might
+need to update ``mach-nix`` itself and the `pypi-deps-db
+<https://github.com/DavHau/pypi-deps-db>`_ version in use (see
+``etc/nix/flake.nix:inputs.machnix`` and ``machnixFor.pypiDataRev``).
+
+Non-Python dependencies are curated in
+``etc/nix/flake.nix:vulnerablecode.propagatedBuildInputs``.
+
+
+
 Run Tests
 ---------
 

etc/nix/default.nix

@@ -0,0 +1,4 @@
+(import (fetchTarball
+  "https://github.com/edolstra/flake-compat/archive/master.tar.gz") {
+    src = ./.;
+  }).defaultNix

etc/nix/flake.nix

@@ -0,0 +1,153 @@
+{
+  description =
+    "Vulnerablecode - A free and open vulnerabilities database and the packages they impact.";
+
+  inputs.nixpkgs = {
+    type = "github";
+    owner = "NixOS";
+    repo = "nixpkgs";
+    ref = "20.09";
+  };
+
+  inputs.machnix = {
+    type = "github";
+    owner = "DavHau";
+    repo = "mach-nix";
+    ref = "3.1.1";
+  };
+
+  outputs = { self, nixpkgs, machnix }:
+    let
+
+      vulnerablecode-src = ./../..;
+
+      # Extract version from setup.py.
+      version = builtins.head (builtins.match ''.*version=["']?([^"',]+).*''
+        (builtins.readFile (vulnerablecode-src + "/setup.py")));
+
+      # Common shell code.
+      libSh = ./lib.sh;
+
+      # System types to support.
+      supportedSystems = [ "x86_64-linux" ];
+
+      # Helper function to generate an attrset '{ x86_64-linux = f "x86_64-linux"; ... }'.
+      forAllSystems = f:
+        nixpkgs.lib.genAttrs supportedSystems (system: f system);
+
+      # Nixpkgs instantiated for supported system types.
+      nixpkgsFor = forAllSystems (system:
+        import nixpkgs {
+          inherit system;
+          overlays = [ self.overlay ];
+        });
+
+      # mach-nix instantiated for supported system types.
+      machnixFor = forAllSystems (system:
+        import machnix {
+          pkgs = (nixpkgsFor.${system}).pkgs;
+          python = "python38";
+
+          # Pin pypi repo to a specific commit which includes all necessary
+          # Python deps. The default version is updated with every mach-nix
+          # release might be be sufficient for newer releases.
+          # The corresponding sha256 hash can be obtained with:
+          # $ nix-prefetch-url --unpack https://github.com/DavHau/pypi-deps-db/tarball/<pypiDataRev>
+          pypiDataRev = "c86b4490a7d838bd54a2d82730455e96c6e4eb14";
+          pypiDataSha256 =
+            "0al490gi0qda1nkb9289z2msgpc633rv5hn3w5qihkl1rh88dmjd";
+        });
+
+    in {
+
+      # A Nixpkgs overlay.
+      overlay = final: prev:
+        with final.pkgs; {
+
+          pythonEnv = machnixFor.${system}.mkPython {
+            requirements =
+              builtins.readFile (vulnerablecode-src + "/requirements.txt");
+          };
+
+          vulnerablecode = stdenv.mkDerivation {
+            inherit version;
+            name = "vulnerablecode-${version}";
+            src = vulnerablecode-src;
+            dontConfigure = true; # do not use ./configure
+            propagatedBuildInputs = [ pythonEnv postgresql ];
+
+            postPatch = ''
+              # Make sure the pycodestyle binary in $PATH is used.
+              substituteInPlace vulnerabilities/tests/test_basics.py \
+                --replace 'join(bin_dir, "pycodestyle")' '"pycodestyle"'
+            '';
+
+            installPhase = ''
+              cp -r . $out
+            '';
+          };
+
+        };
+
+      # Provide a nix-shell env to work with vulnerablecode.
+      devShell = forAllSystems (system:
+        with nixpkgsFor.${system};
+        mkShell {
+          # will be available as env var in `nix develop` / `nix-shell`.
+          VULNERABLECODE_INSTALL_DIR = vulnerablecode;
+          buildInputs = [ vulnerablecode ];
+          shellHook = ''
+            alias vulnerablecode-manage.py=${vulnerablecode}/manage.py
+          '';
+        });
+
+      # Provide some packages for selected system types.
+      packages = forAllSystems
+        (system: { inherit (nixpkgsFor.${system}) vulnerablecode; });
+
+      # The default package for 'nix build'.
+      defaultPackage =
+        forAllSystems (system: self.packages.${system}.vulnerablecode);
+
+      # Tests run by 'nix flake check' and by Hydra.
+      checks = forAllSystems (system: {
+        inherit (self.packages.${system}) vulnerablecode;
+
+        vulnerablecode-test = with nixpkgsFor.${system};
+          stdenv.mkDerivation {
+            name = "${vulnerablecode.name}-test";
+
+            buildInputs = [ wget vulnerablecode ];
+
+            # Used by pygit2.
+            # See https://github.com/NixOS/nixpkgs/pull/72544#issuecomment-582674047.
+            SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+
+            unpackPhase = "true";
+
+            buildPhase = ''
+              source ${libSh}
+              initPostgres $(pwd)
+              export DJANGO_DEV=1
+              ${vulnerablecode}/manage.py migrate
+            '';
+
+            doCheck = true;
+            checkPhase = ''
+              # Run pytest on the installed version. A running postgres
+              # database server is needed.
+              (cd ${vulnerablecode} && pytest)
+
+              # Launch the webserver and call the API.
+              ${vulnerablecode}/manage.py runserver &
+              sleep 2
+              wget http://127.0.0.1:8000/api/
+              kill %1 # kill background task (i.e. webserver)
+            '';
+
+            installPhase =
+              "mkdir -p $out"; # make this derivation return success
+          };
+      });
+    };
+}

etc/nix/lib.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Setup postgres; see the README for the latest instructions.
+#
+# $RUNDIR is used to prevent postgres from accessings its default run dir at
+# /run/postgresql. See
+# https://github.com/NixOS/nixpkgs/issues/83770#issuecomment-607992517
+function initPostgres() {
+  ROOTDIR=$1
+  DATADIR=$ROOTDIR/pgdata
+  RUNDIR=$ROOTDIR/run
+  ENCODING="UTF-8"
+  mkdir -p "$RUNDIR"
+  initdb -D "$DATADIR" -E $ENCODING
+  pg_ctl -D "$DATADIR" -o "-k $RUNDIR" -l "$DATADIR/logfile" start
+  createuser --host "$RUNDIR" --no-createrole --no-superuser --login --inherit --createdb vulnerablecode
+  createdb   --host "$RUNDIR" -E $ENCODING --owner=vulnerablecode --user=vulnerablecode --port=5432 vulnerablecode
+}

etc/nix/shell.nix

@@ -0,0 +1,3 @@
+(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
+  src = ./.;
+}).shellNix