Remove old code and organise deps

Signed-off-by: Shivam Sandbhor <[email protected]>

.github/workflows/main.yml

@@ -27,6 +27,7 @@ jobs:
         run: |
           sudo apt install  python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev postgresql ncat
           python -m pip install --upgrade pip 
+          pip install -r requirements-dev.txt
 
       - uses: syphar/restore-virtualenv@v1
         id: cache-virtualenv

.github/workflows/upstream_test.yml

@@ -38,6 +38,7 @@ jobs:
           sudo apt install  python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev 
           python -m pip install --upgrade pip 
           pip install -r requirements.txt
+          pip install -r requirements-dev.txt
           pip install pytest-xdist
     
       - name: Run tests

README.rst

@@ -149,13 +149,8 @@ Create a virtualenv, install dependencies, generate static files and run the dat
     python3 -m venv venv
     source venv/bin/activate
     pip install -r requirements.txt
-    DJANGO_DEV=1 python manage.py collectstatic
-    DJANGO_DEV=1 python manage.py migrate
-
-The environment variable ``DJANGO_DEV`` is used to load settings suitable for
-development,  defined in ``vulnerablecode/dev.py``. If you
-don't want to type it every time use ``export DJANGO_DEV=1`` instead.
-Do not use `DJANGO_DEV` in a production environment.
+    python manage.py collectstatic
+    python manage.py migrate
 
 
 For a production mode, an environment variable named ``SECRET_KEY`` needs to be
@@ -211,11 +206,11 @@ Non-Python dependencies are curated in::
 
 Run Tests
 ---------
-
+Make sure to install dev dependencies by running ``pip install -r requirements-dev.txt``
 Use these commands to run code style checks and the test suite::
 
     black -l 100 --check .
-    DJANGO_DEV=1 python -m pytest
+    python -m pytest
 
 
 Data import
@@ -233,23 +228,23 @@ for instructions on how to obtain your GitHub token.
 
 To run all data importers use::
 
-    DJANGO_DEV=1 python manage.py import --all
+    python manage.py import --all
 
 To list available importers use::
 
-    DJANGO_DEV=1 python manage.py import --list
+    python manage.py import --list
 
 To run specific importers::
 
-    DJANGO_DEV=1 python manage.py import rust npm 
+    python manage.py import rust npm 
 
 
 REST API access
 ---------------
 
 Start the webserver::
 
-    DJANGO_DEV=1 python manage.py runserver
+    python manage.py runserver
 
 
 For full documentation about API endpoints use this URL::

requirements-dev.txt

@@ -0,0 +1,3 @@
+pytest
+ipython
+black

requirements.txt

@@ -1,61 +1,17 @@
-aiohttp==3.7.4
-asgiref==3.2.7
-attrs==20.3.0
-backcall==0.1.0
-beautifulsoup4==4.7.1
-binaryornot==0.4.4
-cached-property==1.5.1
-cffi==1.14.0
-contextlib2==0.5.5
-decorator==4.4.2
-univers==21.4.16.6
-dj-database-url==0.4.2
-Django==3.0.14
-django-filter==2.2.0
-djangorestframework==3.11.2
-django-widget-tweaks==1.4.8
-drf-spectacular==0.13.0
-gunicorn==19.7.1
-importlib-metadata==1.3.0
-ipython==7.13.0
-ipython-genutils==0.2.0
-jedi==0.17.0
-lxml==4.6.3
-more-itertools==8.0.2
-packageurl-python==0.9.3
-packaging==20.9
-parso==0.7.0
-pexpect==4.8.0
-pickleshare==0.7.5
-pluggy==0.13.1
-prompt-toolkit==3.0.5
-black==20.8b1
-psycopg2==2.8.4
-ptyprocess==0.6.0
-py==1.10.0
-pycparser==2.20
-gitpython==3.1.14
-Pygments==2.7.4
-pyparsing==2.4.5
-pytest==6.2.3
-pytest-django==4.1.0
-django-widget-tweaks==1.4.8 
-pytest-dependency==0.4.0
-pytest-django==4.1.0
-pytest-mock==1.13.0
-python-dateutil==2.8.1
-pytz==2019.3
-schema==0.7.4
-six==1.13.0
-soupsieve==1.9.5
-sqlparse==0.3.0
-tqdm==4.41.1
-traitlets==4.3.3
-wcwidth==0.1.7
-whitenoise==5.0.1
-zipp==0.6.0
-requests==2.23.0
-toml==0.10.2
-PyYAML==5.4
-freezegun==1.1.0
-saneyaml==0.5.2
+Django>=3.2.4
+psycopg2>=2.8.6
+requests>=2.25.1
+aiohttp>=3.7.4.post0
+djangorestframework>=3.12.4
+django-filter>=2.4.0
+django-widget-tweaks>=1.4.8
+drf-spectacular>=0.17.1
+packageurl-python>=0.9.4
+binaryornot>=0.4.4
+GitPython>=3.1.17
+univers>=21.4.16.6
+saneyaml>=0.5.2
+beautifulsoup4>=4.9.3
+python-dateutil>=2.8.1
+toml>=0.10.2
+

vulnerabilities/import_runner.py

@@ -24,13 +24,9 @@
 import dataclasses
 import datetime
 import logging
-from collections import Counter
 from itertools import chain
-import traceback
-from typing import Set
 from typing import Tuple
 
-from django.core import serializers
 from django.db import transaction
 
 from vulnerabilities import models

vulnerabilities/importers/alpine_linux.py

@@ -20,20 +20,14 @@
 #  for any legal advice.
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
-from re import IGNORECASE
 from typing import Any
-from typing import Iterable
 from typing import List
 from typing import Mapping
 from typing import Set
 
 import requests
 import saneyaml
 from bs4 import BeautifulSoup
-from packageurl import PackageURL
-from schema import Or
-from schema import Regex
-from schema import Schema
 
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
@@ -43,40 +37,6 @@
 BASE_URL = "https://secdb.alpinelinux.org/"
 
 
-# def validate_schema(advisory_dict):
-#     scheme = {
-#         "distroversion": Regex(r"v\d.\d*"),
-#         "reponame": str,
-#         "archs": list,
-#         "packages": [
-#             {
-#                 "pkg": {
-#                     "name": str,
-#                     "secfixes": {
-#                         str: Or(
-#                             [
-#                                 Or(
-#                                     Regex(r"CVE.\d+-\d+", flags=IGNORECASE),
-#                                     Regex(r"XSA-\d{3}"),
-#                                     Regex(r"ZBX-\d{4}"),
-#                                     Regex(r"wnpa-sec-\d{4}-\d{2}"),
-#                                     Regex(r"GHSA-.{4}-.{4}-.{4}"),
-#                                 )
-#                             ],
-#                             "",
-#                             # FIXME: Remove the None when below issue gets fixed
-#                             # https://gitlab.alpinelinux.org/alpine/infra/alpine-secdb/-/issues/1
-#                             None,
-#                         ),
-#                     },
-#                 }
-#             }
-#         ],
-#         object: object,
-#     }
-#     Schema(scheme).validate(advisory_dict)
-
-
 class AlpineDataSource(DataSource):
     @staticmethod
     def fetch_advisory_links():

vulnerabilities/importers/archlinux.py

@@ -29,7 +29,6 @@
 from urllib.request import urlopen
 
 from packageurl import PackageURL
-from schema import Regex, Schema, Or
 
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
@@ -40,23 +39,6 @@
 from vulnerabilities.severity_systems import scoring_systems
 
 
-def validate_schema(advisory_dict):
-    scheme = {
-        "advisories": list,
-        "affected": str,
-        "fixed": Or(None, str),
-        "issues": [Regex(r"CVE-\d+-\d+")],
-        "name": str,
-        "packages": [str],
-        "status": str,
-        "ticket": object,
-        "type": str,
-        "severity": str,
-    }
-
-    Schema(scheme).validate(advisory_dict)
-
-
 @dataclasses.dataclass
 class ArchlinuxConfiguration(DataSourceConfiguration):
     archlinux_tracker_url: str
@@ -69,9 +51,6 @@ class ArchlinuxDataSource(DataSource):
     def __enter__(self):
         self._api_response = self._fetch()
 
-        for record in self._api_response:
-            validate_schema(record)
-
     def updated_advisories(self) -> Set[Advisory]:
         advisories = []
 

vulnerabilities/importers/debian.py

@@ -22,7 +22,6 @@
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
 import dataclasses
-import json
 from dateutil import parser as dateparser
 from typing import Any
 from typing import List
@@ -31,10 +30,6 @@
 
 import requests
 from packageurl import PackageURL
-from schema import Optional
-from schema import Or
-from schema import Regex
-from schema import Schema
 
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
@@ -43,41 +38,6 @@
 from vulnerabilities.helpers import nearest_patched_package
 
 
-def validate_schema(advisory_dict):
-
-    deb_versions = [
-        "bullseye",
-        "bullseye-security",
-        "buster",
-        "buster-security",
-        "sid",
-        "stretch",
-        "stretch-security",
-        "jessie",
-        "jessie-security",
-    ]
-    scheme = {
-        str: {
-            Or(Regex(r"CVE-\d+-\d+"), Regex(r"TEMP-.+-.+")): {
-                "releases": {
-                    Or(*deb_versions): {
-                        "repositories": {Or(*deb_versions): str},
-                        "status": str,
-                        "urgency": str,
-                        Optional("fixed_version"): str,
-                        Optional(str): object,
-                    }
-                },
-                Optional("description"): str,
-                Optional("debianbug"): int,
-                Optional(str): object,
-            }
-        }
-    }
-
-    Schema(scheme).validate(advisory_dict)
-
-
 @dataclasses.dataclass
 class DebianConfiguration(DataSourceConfiguration):
     debian_tracker_url: str
@@ -90,7 +50,6 @@ class DebianDataSource(DataSource):
     def __enter__(self):
         if self.response_is_new():
             self._api_response = self._fetch()
-            validate_schema(self._api_response)
 
         else:
             self._api_response = {}

vulnerabilities/importers/safety_db.py

@@ -38,9 +38,6 @@
 from univers.versions import PYPIVersion
 from univers.versions import InvalidVersion
 from packageurl import PackageURL
-from schema import Or
-from schema import Regex
-from schema import Schema
 
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
@@ -52,21 +49,6 @@
 logger = logging.getLogger(__name__)
 
 
-def validate_schema(advisory_dict):
-
-    scheme = [
-        {
-            "advisory": str,
-            "cve": Or(None, str),
-            "id": Regex(r"^pyup.io-\d"),
-            "specs": list,
-            "v": str,
-        }
-    ]
-
-    Schema(scheme).validate(advisory_dict)
-
-
 @dataclasses.dataclass
 class SafetyDbConfiguration(DataSourceConfiguration):
     url: str
@@ -104,13 +86,6 @@ def updated_advisories(self) -> Set[Advisory]:
                 # Skip it. The 'cumin' entry is wrong
                 continue
 
-            try:
-                validate_schema(self._api_response[package_name])
-
-            except Exception as e:
-                logger.error(e)
-                continue
-
             all_package_versions = self.versions.get(package_name)
             if not len(all_package_versions):
                 # PyPi does not have data about this package, we skip these

vulnerabilities/lib_oval.py

@@ -965,7 +965,7 @@ def fromStandaloneFile(path):
 
     @staticmethod
     def getElementTypeFromOvalID(ovalid):
-        """ Gets element type from OVAL id. """
+        """Gets element type from OVAL id."""
 
         if not ovalid or ovalid is None:
             raise ValueError("No OVAL ID given")