Initial work for full end to end conda-store-server tests (#111)

* Initial work for full end to end conda-store-server tests * Adding cypress tests * Try to update conda channels within task * Black formatting * Test login button * Cross origin issues when trying to login via jupyterhub auth * improve usability of the docker button (#110) added a pointer + tooltip fixed namespace field in the URL * Adding new prefix path to tests * Scheduling watch paths to run immediately * Black formatting * Extending tests Co-authored-by: Jaime Rodríguez-Guerra <[email protected]>
conda-incubator · Aug 20, 2021 · 760317b · 760317b
1 parent 520c7f3
commit 760317b
Show file tree

Hide file tree

Showing 13 changed files with 4,295 additions and 35 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -28,6 +28,7 @@ jobs:
       - name: Install Dependencies
         run: |
           pip install .[dev]
+          sudo apt install wait-for-it -y
       - name: Black Formatting
         run: |
           black --version
@@ -37,24 +38,44 @@ jobs:
         run: |
           flake8 --version
           flake8
+      - name: Deploy docker-compose
+        run: |
+          docker-compose up -d
+          docker ps
 
-  build-docker-image:
-    name: 'Build docker images'
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        docker-image:
-          - conda-store
-          - conda-store-server
-    steps:
-      - name: 'Checkout Infrastructure'
-        uses: actions/checkout@v2
-      - name: Docker Build, Tag, Push Image
-        env:
-          IMAGE_NAME: quansight/${{ matrix.docker-image }}
-          IMAGE_TAG: ${{ github.sha }}
+          wait-for-it localhost:5432 # postgresql
+          wait-for-it localhost:9000 # minio
+          wait-for-it localhost:8000 # jupyterhub
+          wait-for-it localhost:5000 # conda-store-server
+
+      ### CYPRESS TESTS
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - name: npm version
         run: |
-          docker build -t $IMAGE_NAME:$IMAGE_TAG ${{ matrix.docker-image }}
+          npm --version
+      - name: Install Cypress dependencies
+        run: |
+          sudo apt-get -y update
+          sudo apt-get install -y \
+               libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev \
+               libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb
+      - name: Cypress run
+        uses: cypress-io/github-action@v2
+        env:
+          CYPRESS_BASE_URL: http://localhost:5000/
+        with:
+          working-directory: tests/e2e/
+      - name: Save Cypress screenshots and videos
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: e2e-cypress
+          path: |
+            ./tests/e2e/cypress/screenshots/
+            ./tests/e2e/cypress/videos/
 
   test-conda-store:
     name: 'Test conda-store'
@@ -79,3 +100,21 @@ jobs:
         run: |
           yarn
           pip install .
+
+  build-docker-image:
+    name: 'Build docker images'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        docker-image:
+          - conda-store
+          - conda-store-server
+    steps:
+      - name: 'Checkout Infrastructure'
+        uses: actions/checkout@v2
+      - name: Docker Build, Tag, Push Image
+        env:
+          IMAGE_NAME: quansight/${{ matrix.docker-image }}
+          IMAGE_TAG: ${{ github.sha }}
+        run: |
+          docker build -t $IMAGE_NAME:$IMAGE_TAG ${{ matrix.docker-image }}
diff --git a/conda-store-server/conda_store_server/app.py b/conda-store-server/conda_store_server/app.py
@@ -199,9 +199,6 @@ def ensure_conda_channels(self):
                 self.db.add(conda_channel)
                 self.db.commit()
 
-        for channel in api.list_conda_channels(self.db):
-            channel.update_packages(self.db)
-
     def register_environment(
         self, specification: dict, namespace: str = None, force_build=False
     ):
@@ -270,6 +267,7 @@ def create_build(self, namespace_id: int, specification_sha256: str):
 
         (
             tasks.task_update_storage_metrics.si()
+            | tasks.task_update_conda_channels.si()
             | tasks.task_build_conda_environment.si(build.id)
             | tasks.task_build_conda_env_export.si(build.id)
             | tasks.task_build_conda_pack.si(build.id)

diff --git a/conda-store-server/conda_store_server/orm.py b/conda-store-server/conda_store_server/orm.py
@@ -199,17 +199,18 @@ def update_packages(self, db):
             # nothing to update
             return
 
-        existing_sha256 = {
-            _[0]
-            for _ in db.query(CondaPackage.sha256)
-            .filter(CondaPackage.channel_id == self.id)
-            .all()
-        }
-
         for architecture in repodata:
             packages = list(repodata[architecture]["packages"].values())
+
+            existing_architecture_sha256 = {
+                _[0]
+                for _ in db.query(CondaPackage.sha256)
+                .filter(CondaPackage.channel_id == self.id)
+                .filter(CondaPackage.subdir == architecture)
+                .all()
+            }
             for package in packages:
-                if package["sha256"] not in existing_sha256:
+                if package["sha256"] not in existing_architecture_sha256:
                     db.add(
                         CondaPackage(
                             build=package["build"],
@@ -228,6 +229,9 @@ def update_packages(self, db):
                             channel_id=self.id,
                         )
                     )
+                    existing_architecture_sha256.add(package["sha256"])
+            db.commit()
+
         self.last_update = datetime.datetime.utcnow()
         db.commit()
 
@@ -243,6 +247,7 @@ class CondaPackage(Base):
             "version",
             "build",
             "build_number",
+            "sha256",
             name="_conda_package_uc",
         ),
     )

diff --git a/conda-store-server/conda_store_server/server/app.py b/conda-store-server/conda_store_server/server/app.py
@@ -118,10 +118,14 @@ def after_request_function(response):
         app.register_blueprint(app_auth, url_prefix=self.url_prefix)
 
         app.conda_store.ensure_namespace()
-        app.conda_store.ensure_directories()
-        app.conda_store.configuration.update_storage_metrics(
-            app.conda_store.db, app.conda_store.store_directory
-        )
         app.conda_store.ensure_conda_channels()
 
+        # schedule tasks
+        app.conda_store.celery_app
+
+        from conda_store_server.worker import tasks
+
+        (tasks.task_watch_paths.si()).apply_async()
+        (tasks.task_update_storage_metrics.si()).apply_async()
+
         app.run(debug=True, host=self.address, port=self.port)
diff --git a/conda-store-server/conda_store_server/server/auth.py b/conda-store-server/conda_store_server/server/auth.py
@@ -170,7 +170,7 @@ class Authentication(LoggingConfigurable):
     login_html = Unicode(
         """
 <div class="text-center">
-    <form class="form-signin" method="POST">
+    <form class="form-signin" method="POST" id="login">
         <h1 class="h3 mb-3 fw-normal">Please sign in</h1>
         <div class="form-floating">
             <input name="username" class="form-control" id="floatingInput" placeholder="Username">
@@ -413,7 +413,7 @@ class GenericOAuthAuthentication(Authentication):
     )
     login_html = Unicode(
         """
-<div class="text-center">
+<div id="login" class="text-center">
     <h1 class="h3 mb-3 fw-normal">Please sign in via OAuth</h1>
     <a class="w-100 btn btn-lg btn-primary" href="{authorization_url}">Sign in with OAuth</a>
 </div>
@@ -534,7 +534,7 @@ def _user_data_key_default(self):
     @default("login_html")
     def _login_html_default(self):
         return """
-<div class="text-center">
+<div id="login" class="text-center">
     <h1 class="h3 mb-3 fw-normal">Please sign in via OAuth</h1>
     <a class="w-100 btn btn-lg btn-primary" href="{authorization_url}">Sign in with GitHub</a>
 </div>
@@ -570,7 +570,7 @@ def _user_data_key_default(self):
     @default("login_html")
     def _login_html_default(self):
         return """
-<div class="text-center">
+<div id="login" class="text-center">
     <h1 class="h3 mb-3 fw-normal">Please sign in via OAuth</h1>
     <a class="w-100 btn btn-lg btn-primary" href="{authorization_url}">Sign in with JupyterHub</a>
 </div>

diff --git a/conda-store-server/conda_store_server/worker/app.py b/conda-store-server/conda_store_server/worker/app.py
@@ -42,6 +42,8 @@ def start(self):
             "worker",
             "--loglevel=INFO",
             "--beat",
+            "--concurrency",
+            "1",
         ]
         self.conda_store.ensure_directories()
         self.conda_store.celery_app.worker_main(argv)
diff --git a/conda-store-server/conda_store_server/worker/tasks.py b/conda-store-server/conda_store_server/worker/tasks.py
@@ -25,6 +25,8 @@ def task_watch_paths():
                 specification=yaml.safe_load(f), namespace="filesystem"
             )
 
+    worker.conda_store.session_factory.remove()
+
 
 @task(name="task_update_storage_metrics")
 def task_update_storage_metrics():
@@ -33,6 +35,8 @@ def task_update_storage_metrics():
         conda_store.db, conda_store.store_directory
     )
 
+    conda_store.session_factory.remove()
+
 
 @task(name="task_update_conda_channels")
 def task_update_conda_channels():
@@ -43,33 +47,39 @@ def task_update_conda_channels():
     for channel in api.list_conda_channels(conda_store.db):
         channel.update_packages(conda_store.db)
 
+    conda_store.session_factory.remove()
+
 
 @task(name="task_build_conda_environment")
 def task_build_conda_environment(build_id):
     conda_store = create_worker().conda_store
     build = api.get_build(conda_store.db, build_id)
     build_conda_environment(conda_store, build)
+    conda_store.session_factory.remove()
 
 
 @task(name="task_build_conda_env_export")
 def task_build_conda_env_export(build_id):
     conda_store = create_worker().conda_store
     build = api.get_build(conda_store.db, build_id)
     build_conda_env_export(conda_store, build)
+    conda_store.session_factory.remove()
 
 
 @task(name="task_build_conda_pack")
 def task_build_conda_pack(build_id):
     conda_store = create_worker().conda_store
     build = api.get_build(conda_store.db, build_id)
     build_conda_pack(conda_store, build)
+    conda_store.session_factory.remove()
 
 
 @task(name="task_build_conda_docker")
 def task_build_conda_docker(build_id):
     conda_store = create_worker().conda_store
     build = api.get_build(conda_store.db, build_id)
     build_conda_docker(conda_store, build)
+    conda_store.session_factory.remove()
 
 
 @task(name="task_update_environment_build")
@@ -83,6 +93,7 @@ def task_update_environment_build(environment_id):
     )
 
     utils.symlink(conda_prefix, environment_prefix)
+    conda_store.session_factory.remove()
 
 
 @task(name="task_delete_build")
@@ -109,3 +120,4 @@ def task_delete_build(build_id):
         conda_store.storage.delete(conda_store.db, build_id, build_artifact.key)
 
     conda_store.db.commit()
+    conda_store.session_factory.remove()
diff --git a/tests/e2e/.gitignore b/tests/e2e/.gitignore
@@ -0,0 +1 @@
+node_modules
diff --git a/tests/e2e/cypress.json b/tests/e2e/cypress.json
@@ -0,0 +1,3 @@
+{
+  "chromeWebSecurity": false
+}
diff --git a/tests/e2e/cypress/.gitignore b/tests/e2e/cypress/.gitignore
@@ -0,0 +1,3 @@
+cypress/videos/
+cypress/screenshots/
+fixtures/
diff --git a/tests/e2e/cypress/integration/main.js b/tests/e2e/cypress/integration/main.js
@@ -0,0 +1,48 @@
+describe('First Test', () => {
+  it('Check conda-store home page', () => {
+    // display home page without login
+    cy.visit('/conda-store/');
+
+    // visit login page
+    cy.visit('/conda-store/login/')
+
+    // click on sign in with jupyterhub
+    // login through jupyterhub oauth server
+    cy.get('#login > a')
+      .should('contain', 'Sign in with JupyterHub')
+      .click();
+
+    // fill in username and password and submit
+    cy.get('#username_input')
+      .type('conda-store-test');
+
+    cy.get('#password_input')
+      .type('test');
+
+    // for some reason this does not
+    // respect redirect_uri
+    cy.get('form').submit();
+
+    // visit login page
+    cy.visit('/conda-store/login/')
+
+    // click on sign in with jupyterhub
+    // login through jupyterhub oauth server
+    cy.get('#login > a')
+      .should('contain', 'Sign in with JupyterHub')
+      .click();
+
+    // jupyterhub authorize access
+    cy.get('form > input').click()
+    cy.url().should('include', 'user')
+
+    // visit home page again
+    cy.get('a.navbar-brand').click()
+
+    // visit environment
+    cy.get('h5.card-title > a').contains('filesystem/python-numpy-env').click()
+
+    // visit build
+    cy.get('li.list-group-item > a').contains('Build').click()
+  })
+})