From 5c313bb63ea0332e3d47d72d0bcb7b1cd6b8d8ea Mon Sep 17 00:00:00 2001
From: Shivam Sandbhor <shivam.sandbhor@gmail.com>
Date: Sat, 13 Mar 2021 12:00:26 +0530
Subject: [PATCH] Add tests for checking upstream data sources

---
 .github/workflows/main.yml             |  2 +-
 .github/workflows/upstream_test.yml    | 51 ++++++++++++++++++++++++++
 vulnerabilities/tests/test_upstream.py | 18 +++++++++
 3 files changed, 70 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/upstream_test.yml
 create mode 100644 vulnerabilities/tests/test_upstream.py

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 61de40e28..08c0aec46 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -38,7 +38,7 @@ jobs:
           pip install -r requirements.txt
     
       - name: Run tests
-        run: pytest
+        run: pytest -v -m "not webtest"
         env:
           # The hostname, username used to communicate with the PostgreSQL service container
           POSTGRES_HOST: localhost
diff --git a/.github/workflows/upstream_test.yml b/.github/workflows/upstream_test.yml
new file mode 100644
index 000000000..30cbcdd90
--- /dev/null
+++ b/.github/workflows/upstream_test.yml
@@ -0,0 +1,51 @@
+on:
+  workflow_dispatch: # allow manual execution
+  push:
+  schedule:
+    # run on every 9 o'clock
+    - cron: '0 9 * * *'
+
+jobs:
+  unit_tests:
+    runs-on: ubuntu-latest
+
+    services:
+      # Label used to access the service container
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: vulnerablecode
+          POSTGRES_DB: vulnerablecode
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          sudo apt install  python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev 
+          python -m pip install --upgrade pip 
+          pip install -r requirements.txt
+    
+      - name: Run tests
+        run: pytest -v -m webtest
+        env:
+          # The hostname, username used to communicate with the PostgreSQL service container
+          POSTGRES_HOST: localhost
+          VC_DB_USER: postgres
+          POSTGRES_PORT: 5432
+          DJANGO_DEV: 1
+          GH_TOKEN: 1 
\ No newline at end of file
diff --git a/vulnerabilities/tests/test_upstream.py b/vulnerabilities/tests/test_upstream.py
new file mode 100644
index 000000000..a875f570e
--- /dev/null
+++ b/vulnerabilities/tests/test_upstream.py
@@ -0,0 +1,18 @@
+import pytest
+from vulnerabilities import importers
+from vulnerabilities.importer_yielder import IMPORTER_REGISTRY
+
+
+@pytest.mark.webtest
+@pytest.mark.parametrize(
+    ("data_source", "config"),
+    ((data["data_source"], data["data_source_cfg"]) for data in IMPORTER_REGISTRY),
+)
+def test_updated_advisories(data_source, config):
+
+    if not data_source == "GitHubAPIDataSource":
+        data_src = getattr(importers, data_source)
+        data_src = data_src(batch_size=1, config=config)
+        with data_src:
+            for i in data_src.updated_advisories():
+                pass