Add scancode.io as a dependency #224

    * Update dependencies

Signed-off-by: Jono Yang <[email protected]>

purldb/settings.py

@@ -26,6 +26,89 @@
 env = environ.Env()
 environ.Env.read_env(str(ENV_FILE))
 
+# ScanCode.io
+
+SCANCODEIO_WORKSPACE_LOCATION = env.str("SCANCODEIO_WORKSPACE_LOCATION", default="var")
+
+SCANCODEIO_CONFIG_DIR = env.str("SCANCODEIO_CONFIG_DIR", default=".scancode")
+
+SCANCODEIO_CONFIG_FILE = env.str(
+    "SCANCODEIO_CONFIG_FILE", default="scancode-config.yml"
+)
+
+SCANCODE_TOOLKIT_RUN_SCAN_ARGS = env.dict("SCANCODE_TOOLKIT_RUN_SCAN_ARGS", default={})
+
+SCANCODEIO_LOG_LEVEL = env.str("SCANCODEIO_LOG_LEVEL", "INFO")
+
+# Set the number of parallel processes to use for ScanCode related scan execution.
+# If the SCANCODEIO_PROCESSES argument is not set, defaults to an optimal number of CPUs
+# available on the machine.
+SCANCODEIO_PROCESSES = env.int("SCANCODEIO_PROCESSES", default=None)
+
+SCANCODEIO_POLICIES_FILE = env.str("SCANCODEIO_POLICIES_FILE", default="policies.yml")
+
+# This setting defines the additional locations ScanCode.io will search for pipelines.
+# This should be set to a list of strings that contain full paths to your additional
+# pipelines directories.
+SCANCODEIO_PIPELINES_DIRS = env.list("SCANCODEIO_PIPELINES_DIRS", default=[])
+
+# Maximum time allowed for a pipeline to complete.
+SCANCODEIO_TASK_TIMEOUT = env.str("SCANCODEIO_TASK_TIMEOUT", default="24h")
+
+# Default to 2 minutes.
+SCANCODEIO_SCAN_FILE_TIMEOUT = env.int("SCANCODEIO_SCAN_FILE_TIMEOUT", default=120)
+
+# Default limit for "most common" entries in QuerySets.
+SCANCODEIO_MOST_COMMON_LIMIT = env.int("SCANCODEIO_MOST_COMMON_LIMIT", default=7)
+
+# PurlDB integration
+
+PURLDB_URL = env.str("PURLDB_URL", default="")
+PURLDB_USER = env.str("PURLDB_USER", default="")
+PURLDB_PASSWORD = env.str("PURLDB_PASSWORD", default="")
+PURLDB_API_KEY = env.str("PURLDB_API_KEY", default="")
+
+# VulnerableCode integration
+
+VULNERABLECODE_URL = env.str("VULNERABLECODE_URL", default="")
+VULNERABLECODE_USER = env.str("VULNERABLECODE_USER", default="")
+VULNERABLECODE_PASSWORD = env.str("VULNERABLECODE_PASSWORD", default="")
+VULNERABLECODE_API_KEY = env.str("VULNERABLECODE_API_KEY", default="")
+
+# Third-party apps
+
+CRISPY_TEMPLATE_PACK = "bootstrap3"
+
+# Job Queue
+
+RQ_QUEUES = {
+    "default": {
+        "HOST": env.str("SCANCODEIO_REDIS_HOST", default="localhost"),
+        "PORT": env.str("SCANCODEIO_REDIS_PORT", default="6379"),
+        "PASSWORD": env.str("SCANCODEIO_REDIS_PASSWORD", default=""),
+        "DEFAULT_TIMEOUT": env.int("SCANCODEIO_REDIS_DEFAULT_TIMEOUT", default=360),
+    },
+}
+
+SCANCODEIO_ASYNC = env.bool("SCANCODEIO_ASYNC", default=False)
+if not SCANCODEIO_ASYNC:
+    for queue_config in RQ_QUEUES.values():
+        queue_config["ASYNC"] = False
+
+# List views pagination, controls the number of items displayed per page.
+# Syntax in .env: SCANCODEIO_PAGINATE_BY=project=10,project_error=10
+SCANCODEIO_PAGINATE_BY = env.dict(
+    "SCANCODEIO_PAGINATE_BY",
+    default={
+        "project": 20,
+        "error": 50,
+        "resource": 100,
+        "package": 100,
+        "dependency": 100,
+        "relation": 100,
+    },
+)
+
 # Security
 
 SECRET_KEY = env.str("SECRET_KEY")
@@ -62,17 +145,25 @@
     'minecode',
     'matchcode',
     'packagedb',
+    'scanpipe',
     # Django built-in
-    "django.contrib.auth",
+    'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.admin',
-    "django.contrib.humanize",
+    'django.contrib.humanize',
     # Third-party apps
+    'crispy_forms',
+    'crispy_bootstrap3',  # required for the djangorestframework browsable API
     'django_filters',
     'rest_framework',
+    'rest_framework.authtoken',
+    'django_rq',
+    'django_probes',
+    'fontawesomefree',
+    'taggit',
 )
 
 MIDDLEWARE = (

purldb/urls.py

@@ -21,7 +21,9 @@
 from matchcode.api import ExactFileIndexViewSet
 from matchcode.api import ExactPackageArchiveIndexViewSet
 from minecode.api import PriorityResourceURIViewSet
-
+from scanpipe.api.views import ProjectViewSet
+from scanpipe.api.views import RunViewSet
+from scancodeio.urls import auth_urlpatterns
 
 api_router = routers.DefaultRouter()
 api_router.register('packages', PackageViewSet)
@@ -33,8 +35,11 @@
 api_router.register('exact_package_archive_index', ExactPackageArchiveIndexViewSet)
 api_router.register('cditems', CDitemViewSet, 'cditems')
 api_router.register('on_demand_queue', PriorityResourceURIViewSet)
+api_router.register('projects', ProjectViewSet)
+api_router.register('runs', RunViewSet)
 
-urlpatterns = [
+urlpatterns = auth_urlpatterns + [
     path('api/', include((api_router.urls, 'api'))),
+    path("", include("scanpipe.urls")),
     path("", RedirectView.as_view(url="api/")),
 ]

requirements-dev.txt

@@ -1,15 +0,0 @@
-aboutcode-toolkit==8.0.0
-black==23.3.0
-et-xmlfile==1.1.0
-exceptiongroup==1.1.1
-execnet==1.9.0
-iniconfig==2.0.0
-mock==5.0.2
-mypy-extensions==1.0.0
-openpyxl==3.1.2
-pathspec==0.11.1
-platformdirs==3.5.1
-pytest==7.3.1
-pytest-django==4.5.2
-pytest-xdist==3.3.1
-tomli==2.0.1

requirements.txt

@@ -1,101 +0,0 @@
-arrow==1.2.3
-asgiref==3.7.2
-attrs==23.1.0
-banal==1.0.6
-beautifulsoup4==4.12.2
-binaryornot==0.4.4
-bitarray==2.6.0
-boolean.py==4.0
-certifi==2023.5.7
-cffi==1.15.1
-chardet==5.1.0
-charset-normalizer==3.1.0
-click==8.1.3
-colorama==0.4.6
-commoncode==31.0.3
-container-inspector==32.0.1
-cryptography==41.0.1
-debian-inspector==31.0.0
-Deprecated==1.2.14
-Django==4.1.2
-django-environ==0.9.0
-django-filter==22.1
-djangorestframework==3.14.0
-dockerfile-parse==2.0.0
-dparse2==0.7.0
-extractcode==31.0.0
-extractcode-7z==16.5.210531
-extractcode-libarchive==3.5.1.210531
-fasteners==0.18
-fingerprints==1.1.0
-ftfy==6.1.1
-ftputil==5.0.4
-gemfileparser2==0.9.3
-gunicorn==20.1.0
-html5lib==1.1
-idna==3.4
-importlib-metadata==6.6.0
-intbitset==3.0.2
-isodate==0.6.1
-jaraco.functools==3.7.0
-javaproperties==0.8.1
-jawa==2.2.0
-Jinja2==3.1.2
-jsonstreams==0.6.0
-license-expression==30.1.1
-lxml==4.9.2
-MarkupSafe==2.1.3
-more-itertools==9.1.0
-natsort==8.2.0
-normality==2.4.0
-packageurl-python==0.11.2
-packaging==23.1
-packvers==21.5
-parameter-expansion-patched==0.3.1
-pdfminer.six==20221105
-pefile==2023.2.7
-pip==22.2.2
-pip-requirements-parser==32.0.1
-pkginfo2==30.0.0
-pluggy==1.0.0
-plugincode==32.0.0
-ply==3.11
-psycopg2==2.9.3
-psycopg2-binary==2.9.3
-publicsuffix2==2.20191221
-pyahocorasick==2.0.0
-pycparser==2.21
-PyGithub==1.56
-pygmars==0.8.0
-Pygments==2.15.1
-PyJWT==2.7.0
-pymaven-patch==0.3.0
-PyNaCl==1.5.0
-pyparsing==3.0.9
-python-dateutil==2.8.2
-pytz==2023.3
-PyYAML==6.0
-rdflib==6.3.2
-reppy2==0.3.6
-requests==2.31.0
-rubymarshal==1.0.3
-saneyaml==0.6.0
-scancode-toolkit==32.0.8
-setuptools==65.3.0
-six==1.16.0
-soupsieve==2.4.1
-spdx-tools==0.7.0rc0
-sqlparse==0.4.4
-text-unidecode==1.3
-toml==0.10.2
-typecode==30.0.1
-typecode-libmagic==5.39.210531
-typing_extensions==4.6.3
-urllib3==2.0.3
-urlpy==0.5
-wcwidth==0.2.6
-webencodings==0.5.1
-wheel==0.37.1
-wrapt==1.15.0
-xmltodict==0.13.0
-zipp==3.15.0

setup.cfg

@@ -39,24 +39,24 @@ install_requires =
     bitarray == 2.6.0
     debian-inspector == 31.0.0
     commoncode == 31.0.3
-    Django == 4.1.2
-    django-environ == 0.9.0
+    Django == 4.2.6
+    django-environ == 0.11.2
     djangorestframework == 3.14.0
-    django-filter == 22.1
-    gunicorn == 20.1.0
+    django-filter == 23.3
+    gunicorn == 21.2.0
     ftputil == 5.0.4
     jawa == 2.2.0
     markdown == 3.5.1
     natsort == 8.2.0
     packageurl-python == 0.11.2
-    psycopg2-binary == 2.9.3
-    psycopg2 == 2.9.3
+    psycopg[binary]==3.1.12
     PyGithub == 1.56
     reppy2 == 0.3.6
     rubymarshal == 1.0.3
     scancode-toolkit[full] == 32.0.8
+    scancodeio == 32.7.0
     urlpy == 0.5
-    matchcode-toolkit >= 1.1.1
+    matchcode-toolkit >= 1.1.3
     univers == 30.11.0
 setup_requires = setuptools_scm[toml] >= 4