Merge pull request #557 from NBISweden/develop

New version of SweFreq with PostgreSQL

.coveragerc

@@ -1,5 +1,5 @@
 [run]
-omit = /usr/local/*,/home/travis/virtualenv/*
+omit = /usr/local/*,/home/travis/virtualenv/*,*venv*
 
 [report]
-omit = /usr/local/*,/home/travis/virtualenv/*
+omit = */__init__.py, */test.py, */test_*.py

.gitignore

@@ -15,3 +15,13 @@ tornado/static/js/app.min.js
 backend/static
 backend/templates
 static
+# importer and config stuff
+mysql-data*
+scripts/importer/downloaded_files
+# docker stuff
+postgres-data
+# local personal things
+personal
+# travis test remnants
+master-schema.sql
+settings.json.tmp

.travis.yml

@@ -1,12 +1,18 @@
 language: python
 python:
-    - "3.5"
+    - "3.6"
 services:
     - docker
 before_install:
     - test/travis_before_install.sh
 install:
     - pip install -r backend/requirements.txt
+    - pip install -r test/requirements.txt
     - pip install coverage coveralls
 script:
     - test/travis_script.sh
+addons:
+  postgresql: "10"
+  apt:
+    packages:
+    - postgresql-client-10

Dockerfile-backend

@@ -1,13 +1,14 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 RUN apt-get update && apt-get install -y \
         python3 \
-        python3-pip \
-        libmysqlclient-dev
+        python3-pip
 
 ADD . /code
+COPY settings_sample.json /settings.json
+RUN sed -i 's/"postgresHost"\s*:.*,/"postgresHost" : "db",/' /settings.json
 WORKDIR /code
 
 RUN pip3 install -r backend/requirements.txt
 
-CMD ["python3", "backend/route.py", "--develop"]
+CMD ["python3", "backend/route.py", "--develop", "--settings_file", "/settings.json"]

Dockerfile-database

@@ -0,0 +1,6 @@
+FROM postgres:10
+
+ENV POSTGRES_DB swefreq
+COPY sql/data_schema.sql /docker-entrypoint-initdb.d/01_data_schema.sql
+COPY sql/user_schema.sql /docker-entrypoint-initdb.d/02_user_schema.sql
+COPY sql/beacon_schema.sql /docker-entrypoint-initdb.d/03_beacon_schema.sql

Dockerfile-frontend-rebuilder

@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 RUN apt-get update && \
     apt-get install -y \
@@ -7,12 +7,12 @@ RUN apt-get update && \
         python3 \
         python3-pip \
         python3-pyinotify \
-        inotify-tools \
-        libmysqlclient-dev && \
+        inotify-tools && \
     update-alternatives --install /usr/bin/python python /usr/bin/python3 5
 
-RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - && \
-    apt-get install -y nodejs
+RUN apt-get install -y \
+        nodejs \
+        npm
 
 ADD . /code
 WORKDIR /code

README.md

@@ -7,76 +7,76 @@ SweFreq - Swedish Frequency database
 Running on a production system
 ------------------------------
 
-The application has only been tested with python 3.5.2. It will most likely work with at least other 3.5 versions.
+The application is targeted at python 3.6. It will most likely work with later versions as well.
 
-`virtualenv` is not a requirement but it will help you to install the application.
+`venv` is not a requirement but it will help you to install the application.
 
 1. Download the repository from [github](https://github.com/NBISweden/swefreq)
 
-2. Credentials
-
-   2.1 Make sure you have registered an Elixir AAI application with the Elixir.
-
-   2.2 (legacy) Create Google OAuth 2.0 credentials with the [Google API
-   Console](https://console.developers.google.com/) for enabling
-   authentication.
+2. Make sure you have registered an Elixir AAI application with the Elixir.
 
 3. Rename the file `settings_sample.json` into `settings.json` and edit all
    the values.
 
-4. Install MySQL. Only tested with 5.6. Note, the schema is incompatible with the
-   latest version of MariaDB.
+4. If you have `venv` and `pip` available you can do the following
+   to install the required Python packages:
+
+   ```
+   python -m venv venv-folder
+   source /path/to/venv-folder/bin/activate             # activate your virtual environment
+   pip install -r /path/to/requrements.txt  # install the required Python packages
+   ```
 
-5. If you have `virtualenv` and `pip` installed then you can do the following
-   to install the required Python packages. Locate the `requirements.txt` file
-   in the `swefreq` repository.
+   * backend: `backend/requirements.txt`
+   * importer: `scripts/importer/requirements.txt`
+   * documentation: `docs/requirements.txt`
 
-    source /path/to/bin/activate             # activate your virtualenv
-    pip install -r /path/to/requrements.txt  # install the required Python packages
 
-6. Create the MySQL database, user and tables with the following command:
+5. Create the PostgreSQL database and its tables with the following command:
 
-       mysql -u root -p < ./sql/swefreq.sql
+   ```
+   psql -U postgres -h 127.0.0.1 -f sql/data_schema.sql
+   psql -U postgres -h 127.0.0.1 -f sql/user_schema.sql
+   ```
 
-   To experience the full site you need to manually add a dataset and a user
-   to the database. Log into the mysql console and enter something like the
-   following, change the values to whatever fits your site (N.B.
-   `[email protected]` is your google id):
+   To experience the full site you need to manually add a dataset and a user to the database.
+   You can use the test data in `test/data/browser_test_data.sql` as reference.
 
-       USE swefreq;
-       INSERT INTO dataset (dataset_pk, name) VALUES (1, "Dataset");
-       INSERT INTO user (user_pk,name, email) VALUES (1, "test", "[email protected]");
-       INSERT INTO dataset_access (dataset_pk, user_pk, is_admin, has_access) VALUES (1,1,1,1);
+6. Add reference data and variants using the import scripts found in `scripts/`, e.g.:
 
+   ```
+   ./manage.sh import --add_reference \
+                      --gencode_version 19 \
+                      --ensembl_version homo_sapiens_core_75_37 \
+                      --assembly_id GRCh37p13 \
+                      --dbnsfp_version 2.9.3 \
+                      --ref_name GRCh37p13
+
+   ./manage.sh import --add_raw_data \
+                      --dataset variant_data \
+                      --version 20190415 \
+                      --variant_file variants/chr22.vcf.gz \
+                      --coverage_file coverage/chr22.coverage.txt.gz
+   ```
 
 ### Start the server
 
-    source /path/to/bin/activate                   # activate your virtualenv
+```
+    source /path/to/bin/activate                   # activate your virtual environment
     python /path/to/route.py
-
+```
 
 Quick development mode
 ----------------------
 
-1. Install docker (and docker-compose in case it's not included in the installation)
-2. Create test database
-
-    1. Initiate a mysql data volume by running `./scripts/create_docker_db_volume_tarball.sh` or 
-         `./scripts/download_and_create_docker_db_volume.sh`
-
-    2. Load mysql dummy data by running `./scripts/load_mysql_dummy_data.sh`
-
-3. Copy `settings_sample.json` into `settings.json` and
-    - Update the credentials for elixir and google oauth.
-        - Elixir/redirectUri: http://localhost:4000/elixir/login
-        - redirectUri: http://localhost:4000/login
-    - Set `mysqlHost` to `db`
-    - Set `mysqlSchema` to `swefreq_test`
-    - Clear `mysqlPasswd` (set it to empty `""`)
-4. Make a symbolic link from `backend/static` to `static`.
-5. Run the server:
+1. Install docker (and docker-compose in case it's not included in the installation) 
+2. Run the server:
+```bash
+docker-compose up
+```
+3. Add test data to db:
 ```bash
-$ docker-compose up
+psql -h localhost -U postgres swefreq -f test/data/browser_test_data.sql
 ```
 
 [travis-badge]: https://travis-ci.org/NBISweden/swefreq.svg?branch=develop