Skip to content

Commit

Permalink
Update docker configuration (#4533)
Browse files Browse the repository at this point in the history
* Update docker configuration

* Update tox, travis

* Pin astroid

* Pin pytest to same version as xpro

* JS tests

* Downgrade pylint_django

* Update test requirements

* Update pytest-django

* First steps on selenium tests

* Fix selenium tests?

* Django test fixes

* Fix selenium?

* Lint

* Update python-dateutil to fix deprecation warning

* Fix selenium?

* Lint

* Fix environment variables for selenium tests

* Fix selenium?

* Fixture

* Update redis python library

* LuaLock was renamed to Lock

* Update beautifulsoup4 to fix deprecation warning

* Deprecation warning

* pip-compile, followed by repinning beautifulsoup4

* Fix deprecation warning

* Deprecation warnings

* Lint

* Pylint

* Fix lock

* Fix test

* Inline py-environment, remove exposed ports

* Pin dateutil to Python 3.7 compatible version to fix deprecation warnings

* Downgrade a couple dependencies to fix incompatibilities

* Lint

* Silence deprecation warnings

* Newline
  • Loading branch information
George Schneeloch authored Mar 4, 2020
1 parent 8373d6d commit a7abe70
Show file tree
Hide file tree
Showing 41 changed files with 188 additions and 248 deletions.
23 changes: 13 additions & 10 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
sudo: false
# Set Ruby as the language so it doesn't download the pip things. Instead, let docker do that.
language: ruby

# xenial for newer docker-compose https://blog.travis-ci.com/2018-11-08-xenial-release
dist: xenial

matrix:
include:
- install:
- language: ruby
install:
- env | grep TRAVIS > .env
- env | grep CI >> .env
- docker-compose -f docker-compose.yml -f docker-compose.travis.yml run celery echo
script: docker-compose -f docker-compose.yml -f docker-compose.travis.yml run web tox
services:
- docker
Expand All @@ -16,6 +17,7 @@ matrix:
- install:
- env | grep TRAVIS > .env
- env | grep CI >> .env
- docker build -t micromasters_watch -f ./Dockerfile-node .
- docker build -t travis-watch -f ./travis/Dockerfile-travis-watch .
- docker run --name travis-watch-container --env-file .env -e NODE_ENV=production -t travis-watch ./webpack_if_prod.sh
- docker cp travis-watch-container:/src/webpack-stats.json .
Expand All @@ -25,12 +27,13 @@ matrix:
- docker
env:
name: Python-Selenium
- install:
- env | grep TRAVIS > .env
- env | grep CI >> .env
- docker build -t travis-watch -f ./travis/Dockerfile-travis-watch .
- language: node_js
before_install:
- curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 1.13.0
- export PATH="$HOME/.yarn/bin:$PATH"
node_js:
- "10.15.1"
script: bash ./travis/js_tests.sh
services:
- docker
cache: yarn
env:
name: JavaScript
16 changes: 16 additions & 0 deletions conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,22 @@ def warnings_as_errors():
message="'async' and 'await' will become reserved keywords in Python 3.7",
category=DeprecationWarning,
)
warnings.filterwarnings(
"ignore",
message=(
"Using or importing the ABCs from 'collections' instead of "
"from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working"
),
category=DeprecationWarning
)
warnings.filterwarnings(
"ignore",
message=(
"Using or importing the ABCs from 'collections' instead of "
"from 'collections.abc' is deprecated, and in 3.8 it will stop working"
),
category=DeprecationWarning
)
# For compatibility modules in various libraries
warnings.filterwarnings(
"ignore",
Expand Down
1 change: 1 addition & 0 deletions dashboard/factories.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
FAKE = faker.Factory.create()


# pylint: disable=arguments-differ
class CachedCertificateFactory(DjangoModelFactory):
"""Factory for Certificate"""
user = SubFactory(UserFactory)
Expand Down
5 changes: 3 additions & 2 deletions dashboard/views_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,9 @@ def test_get_dashboard(self):
assert 'is_edx_data_fresh' in result.data
assert result.data['is_edx_data_fresh'] is True
assert len(result.data['programs']) == 2
assert set([self.program_1.id, self.program_2.id]) == set(
[res_item['id'] for res_item in result.data['programs']])
assert {self.program_1.id, self.program_2.id} == {
res_item['id'] for res_item in result.data['programs']
}

@ddt.data(Instructor, Staff)
@patch('dashboard.api.CachedEdxDataApi.update_cache_if_expired')
Expand Down
2 changes: 1 addition & 1 deletion discussions/factories.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ class DiscussionUserFactory(DjangoModelFactory):
last_sync = Faker('date_time_this_month', before_now=True, after_now=False, tzinfo=UTC)

@classmethod
def create(cls, *args, **kwargs):
def create(cls, *args, **kwargs): # pylint: disable=arguments-differ
"""
Overrides the default .create() method to turn off save signals
"""
Expand Down
11 changes: 1 addition & 10 deletions docker-compose.override.yml
Original file line number Diff line number Diff line change
@@ -1,14 +1,5 @@
version: '2.1'
version: '3.7'
services:
python:
volumes:
- .:/src
- django_media:/var/media
environment:
DEBUG: 'True'
NODE_ENV: 'development'
MICROMASTERS_USE_WEBPACK_DEV_SERVER: 'True'

web:
volumes:
- .:/src
Expand Down
23 changes: 17 additions & 6 deletions docker-compose.selenium.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,23 @@
version: '2.1'
version: '3.7'

services:
selenium:
image: micromasters_python
extends:
service: python
build:
context: .
dockerfile: Dockerfile
environment:
NODE_ENV: 'production'
DEV_ENV: 'True' # necessary to have nginx connect to web container
COVERAGE_DIR: htmlcov
DATABASE_URL: postgres://postgres@db:5432/postgres
MICROMASTERS_SECURE_SSL_REDIRECT: 'False'
MICROMASTERS_DB_DISABLE_SSL: 'True'
ELASTICSEARCH_URL: elastic:9200
CELERY_ALWAYS_EAGER: 'False'
BROKER_URL: redis://redis:6379/4
CELERY_RESULT_BACKEND: redis://redis:6379/4
DOCKER_HOST: ${DOCKER_HOST:-missing}
WEBPACK_DEV_SERVER_HOST: ${WEBPACK_DEV_SERVER_HOST:-localhost}
ELASTICSEARCH_INDEX: 'testindex'
DEBUG: 'False'
ELASTICSEARCH_DEFAULT_PAGE_SIZE: '5'
Expand All @@ -19,8 +32,6 @@ services:
- redis
- hub
- chrome
ports:
- "7000:7000"

hub:
image: selenium/hub:3.141.59-mercury
Expand Down
20 changes: 1 addition & 19 deletions docker-compose.travis.yml
Original file line number Diff line number Diff line change
@@ -1,27 +1,9 @@
version: '2.1'
version: '3.7'
services:
python:
build:
context: .
dockerfile: ./travis/Dockerfile-travis-web

celery:
build:
context: .
dockerfile: ./travis/Dockerfile-travis-web
environment:
# for celery, to avoid ImproperlyConfigured
MAILGUN_URL: 'http://fake.example.com'
MAILGUN_KEY: 'fake'
ELASTICSEARCH_INDEX: 'testindex'
OPEN_DISCUSSIONS_SITE_KEY: 'fake'

web:
build:
context: .
dockerfile: ./travis/Dockerfile-travis-web

selenium:
build:
context: .
dockerfile: ./travis/Dockerfile-travis-web
55 changes: 28 additions & 27 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,24 @@
version: '2.1'
version: '3.7'

x-environment:
&py-environment
DEBUG: 'False'
NODE_ENV: 'production'
DEV_ENV: 'True' # necessary to have nginx connect to web container
COVERAGE_DIR: htmlcov
DATABASE_URL: postgres://postgres@db:5432/postgres
MICROMASTERS_SECURE_SSL_REDIRECT: 'False'
MICROMASTERS_DB_DISABLE_SSL: 'True'
ELASTICSEARCH_URL: elastic:9200
CELERY_ALWAYS_EAGER: 'False'
BROKER_URL: redis://redis:6379/4
CELERY_RESULT_BACKEND: redis://redis:6379/4
EXAMS_SFTP_HOST: sftp
EXAMS_SFTP_USERNAME: odl
EXAMS_SFTP_PASSWORD: 123
DOCKER_HOST: ${DOCKER_HOST:-missing}
WEBPACK_DEV_SERVER_HOST: ${WEBPACK_DEV_SERVER_HOST:-localhost}

services:
db:
image: postgres:11.6
Expand All @@ -24,35 +44,14 @@ services:
links:
- web

python:
web:
build:
context: .
dockerfile: Dockerfile
command: /bin/true
environment:
DEBUG: 'False'
NODE_ENV: 'production'
DEV_ENV: 'True' # necessary to have nginx connect to web container
<< : *py-environment
PORT: 8077
COVERAGE_DIR: htmlcov
DATABASE_URL: postgres://postgres@db:5432/postgres
MICROMASTERS_SECURE_SSL_REDIRECT: 'False'
MICROMASTERS_DB_DISABLE_SSL: 'True'
ELASTICSEARCH_URL: elastic:9200
CELERY_ALWAYS_EAGER: 'False'
BROKER_URL: redis://redis:6379/4
CELERY_RESULT_BACKEND: redis://redis:6379/4
EXAMS_SFTP_HOST: sftp
EXAMS_SFTP_USERNAME: odl
EXAMS_SFTP_PASSWORD: 123
DOCKER_HOST: ${DOCKER_HOST:-missing}
WEBPACK_DEV_SERVER_HOST: ${WEBPACK_DEV_SERVER_HOST:-localhost}
env_file: .env

web:
image: micromasters_python
extends:
service: python
command: >
/bin/bash -c '
sleep 3 &&
Expand Down Expand Up @@ -85,9 +84,11 @@ services:
env_file: .env

celery:
image: micromasters_python
extends:
service: python
build:
context: .
dockerfile: Dockerfile
environment: *py-environment
env_file: .env
command: >
/bin/bash -c '
sleep 3;
Expand Down
2 changes: 1 addition & 1 deletion ecommerce/views_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from django.test import override_settings
from factory.django import mute_signals
import faker
import rest_framework.status as status
from rest_framework import status

from courses.factories import CourseRunFactory
from ecommerce.api import (
Expand Down
2 changes: 1 addition & 1 deletion exams/pearson/readers.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
)


class BaseTSVReader(object):
class BaseTSVReader:
"""
Base class for TSV file readers
Expand Down
2 changes: 1 addition & 1 deletion exams/pearson/writers.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
log = logging.getLogger(__name__)


class BaseTSVWriter(object):
class BaseTSVWriter:
"""
Base class for TSV file writers.
Expand Down
2 changes: 1 addition & 1 deletion exams/pearson/writers_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def get_addr1(profile): # pylint: disable=missing-docstring

addr1_field_mapper = BaseTSVWriter.get_field_mapper(get_addr1)

assert addr1_field_mapper == get_addr1
assert addr1_field_mapper is get_addr1
assert addr1_field_mapper(profile) == profile.address1

with self.assertRaises(TypeError):
Expand Down
2 changes: 1 addition & 1 deletion financialaid/views_pytest_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ def test_view_with_search(self, staff_client, program_data):
# Two users should match the search term - one for first_name, one for last_name
assert len(financial_aid_objects) == 2
assert all(
fin_aid.user.profile.first_name == name_query or fin_aid.user.profile.last_name == name_query
name_query in (fin_aid.user.profile.first_name, fin_aid.user.profile.last_name)
for fin_aid in financial_aid_objects
)

Expand Down
4 changes: 2 additions & 2 deletions grades/api_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,8 @@ def test_get_compute_func(self):
"""
tests for _get_compute_func function
"""
assert api._get_compute_func(self.run_fa) == api._compute_grade_for_fa
assert api._get_compute_func(self.run_no_fa) == api._compute_grade_for_non_fa
assert api._get_compute_func(self.run_fa) is api._compute_grade_for_fa
assert api._get_compute_func(self.run_no_fa) is api._compute_grade_for_non_fa

@patch('grades.api._compute_grade_for_non_fa', autospec=True)
@patch('grades.api._compute_grade_for_fa', autospec=True)
Expand Down
6 changes: 3 additions & 3 deletions grades/tasks_pytest_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,9 @@ def test_generate_course_certificates():
certificates = MicromastersCourseCertificate.objects.filter(course__in=[course, course_2, course_with_exams])
assert certificates.count() == 7
expected_certificate_final_grades = passed_final_grades + final_grades_with_passed_exam
assert set(certificates.values_list('user', flat=True)) == set(
[final_grade.user.id for final_grade in expected_certificate_final_grades]
)
assert set(certificates.values_list('user', flat=True)) == {
final_grade.user.id for final_grade in expected_certificate_final_grades
}


def test_create_combined_final_grade(mocker):
Expand Down
2 changes: 1 addition & 1 deletion mail/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ class AutomaticEmailView(ListAPIView, UpdateModelMixin, GenericViewSet):
serializer_class = AutomaticEmailSerializer
lookup_field = "id"
lookup_url_kwarg = "email_id"
lookup_value_regex = '[-\w.]+' # pylint: disable=anomalous-backslash-in-string
lookup_value_regex = r'[-\w.]+'

def get_queryset(self):
"""Get the queryset which should be serialized"""
Expand Down
2 changes: 1 addition & 1 deletion micromasters/factories.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ class Meta:
class SocialUserFactory(UserFactory):
"""Factory for Users which should also have a social_auth object created for them"""
@classmethod
def create(cls, *args, **kwargs):
def create(cls, *args, **kwargs): # pylint: disable=arguments-differ
"""
Overrides the default .create() method so that a UserSocialAuth records can be created
"""
Expand Down
4 changes: 2 additions & 2 deletions micromasters/locks.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from django.core.cache import caches
from redis.exceptions import LockError
from redis.lock import LuaLock
from redis.lock import Lock as LuaLock

from micromasters.utils import now_in_utc

Expand All @@ -17,7 +17,7 @@ def _get_lock(lock_name, expiration):
expiration (datetime.datetime): The expiration datetime
Returns:
redis.lock.LuaLock: a redis lua-based lock
redis.lock.Lock: a redis lua-based lock
"""
timeout = int((expiration - now_in_utc()).total_seconds())

Expand Down
2 changes: 1 addition & 1 deletion micromasters/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ class FakeSiteMiddleware(MiddlewareMixin):
A mock implementation of `wagtail.core.middleware.SiteMiddleware`
that doesn't make any database calls.
"""
def process_request(self, request): # pylint: disable=missing-docstring,no-self-use
def process_request(self, request): # pylint: disable=missing-docstring,no-self-use,useless-return
request.site = None
return None

Expand Down
4 changes: 2 additions & 2 deletions profiles/factories.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ class Params:
)

@classmethod
def create_batch(cls, *args, **kwargs):
def create_batch(cls, *args, **kwargs): # pylint: disable=arguments-differ
"""
Ensure that signals are muted before running the base create_batch method
"""
Expand All @@ -122,7 +122,7 @@ class SocialProfileFactory(ProfileFactory):
user = SubFactory(SocialUserFactory)

@classmethod
def create(cls, *args, **kwargs):
def create(cls, *args, **kwargs): # pylint: disable=arguments-differ
"""
Overrides the default .create() method to turn off save signals
"""
Expand Down
Loading

0 comments on commit a7abe70

Please sign in to comment.