From 7678187135f95b8c2b87c7b533fafcc60da45d70 Mon Sep 17 00:00:00 2001 From: Hunia Fatima Date: Tue, 19 Nov 2024 15:52:19 +0500 Subject: [PATCH] chore: add enterprise catalog in devstack --- Makefile | 3 ++ docker-compose-host.yml | 10 ++++- docker-compose.yml | 77 +++++++++++++++++++++++++++++++++ docs/service_list.rst | 3 ++ options.mk | 2 +- provision-enterprise_catalog.sh | 32 ++++++++++++++ provision-mysql80.sql | 4 ++ provision.sh | 3 +- provision.sql | 2 + repo.sh | 2 + 10 files changed, 135 insertions(+), 3 deletions(-) create mode 100755 provision-enterprise_catalog.sh diff --git a/Makefile b/Makefile index aba37bc7e2..0ce41f8e61 100644 --- a/Makefile +++ b/Makefile @@ -407,6 +407,9 @@ dev.shell.analyticsapi: dev.shell.insights: docker compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /bin/bash' +dev.shell.enterprise_catalog: + docker exec -it edx.devstack.enterprise.catalog env TERM=$(TERM) bash -c '/bin/bash' + dev.shell.%: ## Run a shell on the specified service's container. docker compose exec $* /bin/bash diff --git a/docker-compose-host.yml b/docker-compose-host.yml index 5941bfbdc2..72a62bb3cf 100644 --- a/docker-compose-host.yml +++ b/docker-compose-host.yml @@ -61,7 +61,15 @@ services: volumes: - ${DEVSTACK_WORKSPACE}/edx-analytics-data-api:/edx/app/analytics_api/analytics_api - ${DEVSTACK_WORKSPACE}/src:/edx/src - + enterprise-catalog: + volumes: + - ${DEVSTACK_WORKSPACE}/enterprise-catalog:/edx/app/enterprise_catalog/enterprise_catalog + enterprise-catalog-worker: + volumes: + - ${DEVSTACK_WORKSPACE}/enterprise-catalog:/edx/app/enterprise_catalog/enterprise_catalog + enterprise-catalog-curations_worker: + volumes: + - ${DEVSTACK_WORKSPACE}/enterprise-catalog:/edx/app/enterprise_catalog/enterprise_catalog # Note that frontends mount `src` to /edx/app/src instead of /edx/src. # See ADR #5 for rationale. frontend-app-account: diff --git a/docker-compose.yml b/docker-compose.yml index a165a6bf22..522aee0303 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -693,6 +693,82 @@ services: aliases: - edx.devstack.xqueue_consumer + enterprise-catalog: + image: edxops/enterprise-catalog-dev + container_name: edx.devstack.enterprise.catalog + hostname: edx.devstack.enterprise.catalog + command: bash -c 'while true; do python /edx/app/enterprise_catalog/enterprise_catalog/manage.py runserver 0.0.0.0:18160; sleep 2; done' + ports: + - "18160:18160" + depends_on: + - memcached + - mysql80 + - enterprise-catalog-worker + networks: + default: + aliases: + - edx.devstack.enterprise.catalog + # Allows attachment to this container using 'docker attach '. + stdin_open: true + tty: true + environment: + CELERY_ALWAYS_EAGER: 'false' + CELERY_BROKER_TRANSPORT: redis + CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379 + CELERY_BROKER_VHOST: 0 + CELERY_BROKER_PASSWORD: password + DJANGO_SETTINGS_MODULE: enterprise_catalog.settings.devstack + ENABLE_DJANGO_TOOLBAR: 1 + DB_HOST: edx.devstack.mysql80 + DB_NAME: enterprise_catalog + DB_PORT: 3306 + DB_USER: catalog001 + DB_PASSWORD: 'password' + + enterprise-catalog-worker: + image: edxops/enterprise-catalog-dev + command: bash -c 'cd /edx/app/enterprise_catalog/enterprise_catalog && celery -A enterprise_catalog worker -Q enterprise_catalog.default -l DEBUG' + container_name: edx.devstack.enterprise.catalog.worker + depends_on: + - mysql80 + environment: + CELERY_ALWAYS_EAGER: 'false' + CELERY_BROKER_TRANSPORT: redis + CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379 + CELERY_BROKER_VHOST: 0 + CELERY_BROKER_PASSWORD: password + DJANGO_SETTINGS_MODULE: enterprise_catalog.settings.devstack + COLUMNS: 80 + hostname: worker.catalog.enterprise + ports: + - "18161:18161" + restart: always + # Allows attachment to this container using 'docker attach '. + stdin_open: true + tty: true + + enterprise-catalog-curations_worker: + image: edxops/enterprise-catalog-dev + command: bash -c 'cd /edx/app/enterprise_catalog/enterprise_catalog && celery -A enterprise_catalog worker -Q enterprise_catalog.curations -l DEBUG' + container_name: enterprise.catalog.curations + depends_on: + - mysql80 + environment: + CELERY_ALWAYS_EAGER: 'false' + CELERY_BROKER_TRANSPORT: redis + CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379 + CELERY_BROKER_VHOST: 0 + CELERY_BROKER_PASSWORD: password + DJANGO_SETTINGS_MODULE: enterprise_catalog.settings.devstack + COLUMNS: 80 + hostname: curations.catalog.enterprise + ports: + - "18162:18162" + restart: always + # Allows attachment to this container using 'docker attach '. + stdin_open: true + tty: true + # ========================================================================== # edX Microfrontends # @@ -910,3 +986,4 @@ volumes: mysql57_data: mysql80_data: redis_data: + enterprise_catalog_mysql8: diff --git a/docs/service_list.rst b/docs/service_list.rst index 092b8028c6..fd750f117a 100644 --- a/docs/service_list.rst +++ b/docs/service_list.rst @@ -63,6 +63,8 @@ Instead of a service name or list, you can also run commands like ``make dev.pro +------------------------------------+-------------------------------------+----------------+--------------+ | `frontend-app-ora-grading`_ | http://localhost:1993 | MFE (React.js) | Extra | +------------------------------------+-------------------------------------+----------------+--------------+ +| `enterprise-catalog`_ | http://localhost:18160/ | Python/Django | Extra | ++------------------------------------+-------------------------------------+----------------+--------------+ Some common service combinations include: @@ -95,3 +97,4 @@ Some common service combinations include: .. _frontend-app-ora-grading: https://github.com/edx/frontend-app-ora-grading .. _insights: https://github.com/edx/edx-analytics-dashboard .. _analyticsapi: https://github.com/edx/edx-analytics-data-api +.. _enterprise-catalog: https://github.com/openedx/enterprise-catalog diff --git a/options.mk b/options.mk index 5a4d9b4e2c..a5adeb6d68 100644 --- a/options.mk +++ b/options.mk @@ -67,7 +67,7 @@ credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-authn+frontend- # Separated by plus signs. # Separated by plus signs. Listed in alphabetical order for clarity. EDX_SERVICES ?= \ -analyticsapi+credentials+cms+cms-worker+cms_watcher+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-learner-dashboard+frontend-app-learner-record+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms-worker+lms_watcher+registrar+registrar-worker+xqueue+xqueue_consumer +analyticsapi+credentials+cms+cms-worker+cms_watcher+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-learner-dashboard+frontend-app-learner-record+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms-worker+lms_watcher+registrar+registrar-worker+xqueue+xqueue_consumer+enterprise_catalog # Services with database migrations. # Should be a subset of $(EDX_SERVICES). diff --git a/provision-enterprise_catalog.sh b/provision-enterprise_catalog.sh new file mode 100755 index 0000000000..820609d02d --- /dev/null +++ b/provision-enterprise_catalog.sh @@ -0,0 +1,32 @@ +name="enterprise_catalog" +port="18160" + +docker-compose up -d + +# Wait for MySQL +echo "Waiting for MySQL" +until docker exec -i enterprise.catalog.mysql mysql -u root -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null +do + printf "." + sleep 1 +done +sleep 5 + +# Run migrations +echo -e "${GREEN}Running migrations for ${name}...${NC}" +docker exec -t enterprise.catalog.app bash -c "cd /edx/app/${name}/${name}/ && make migrate" + +# Create superuser +echo -e "${GREEN}Creating super-user for ${name}...${NC}" +docker exec -t enterprise.catalog.app bash -c "echo 'from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None' | python /edx/app/${name}/${name}/manage.py shell" + +# Provision IDA User in LMS +echo -e "${GREEN}Provisioning ${name}_worker in LMS...${NC}" +docker exec -t edx.devstack.lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user ${name}_worker ${name}_worker@example.com --staff --superuser" + +# Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. +docker exec -t edx.devstack.lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris 'http://localhost:${port}/complete/edx-oauth2/' --client-id '${name}-sso-key' --client-secret '${name}-sso-secret' --scopes 'user_id' ${name}-sso ${name}_worker" +docker exec -t edx.devstack.lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id '${name}-backend-service-key' --client-secret '${name}-backend-service-secret' ${name}-backend-service ${name}_worker" + +# Restart enterprise.catalog app and worker containers +docker-compose restart app worker diff --git a/provision-mysql80.sql b/provision-mysql80.sql index 19b32e0ebd..be903f1d7f 100644 --- a/provision-mysql80.sql +++ b/provision-mysql80.sql @@ -6,6 +6,10 @@ CREATE DATABASE IF NOT EXISTS credentials; CREATE USER IF NOT EXISTS 'credentials001'@'%' IDENTIFIED BY 'password'; GRANT ALL ON credentials.* TO 'credentials001'@'%'; +CREATE DATABASE IF NOT EXISTS enterprise_catalog; +CREATE USER IF NOT EXISTS 'catalog001'@'%' IDENTIFIED BY 'password'; +GRANT ALL ON enterprise_catalog.* TO 'catalog001'@'%'; + CREATE DATABASE IF NOT EXISTS discovery; CREATE USER IF NOT EXISTS 'discov001'@'%' IDENTIFIED BY 'password'; GRANT ALL ON discovery.* TO 'discov001'@'%'; diff --git a/provision.sh b/provision.sh index cde27b6fcd..252a17175d 100755 --- a/provision.sh +++ b/provision.sh @@ -3,7 +3,7 @@ # This script will provision the services specified in the argument list, # or all services if no arguments are provided. # -# Non-existant services will be ignored. +# Non-existent services will be ignored. # Specifying services more than once will cause them to be provisioned more # than once. # @@ -49,6 +49,7 @@ xqueue \ coursegraph \ insights \ analyticsapi \ +enterprise_catalog \ " # What should we provision? diff --git a/provision.sql b/provision.sql index 0b672c9986..94c3fd48c0 100644 --- a/provision.sql +++ b/provision.sql @@ -33,5 +33,7 @@ GRANT ALL ON `reports`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; CREATE DATABASE IF NOT EXISTS `reports_v1`; GRANT ALL ON `reports_v1`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; +CREATE DATABASE IF NOT EXISTS `enterprise_catalog`; +GRANT ALL ON `enterprise_catalog`.* TO 'catalog001'@'%' IDENTIFIED BY 'password'; FLUSH PRIVILEGES; diff --git a/repo.sh b/repo.sh index cb93108f63..84792fb35c 100755 --- a/repo.sh +++ b/repo.sh @@ -36,6 +36,7 @@ repos=( "https://github.com/openedx/frontend-app-publisher.git" "https://github.com/edx/edx-analytics-dashboard.git" "https://github.com/edx/edx-analytics-data-api.git" + "https://github.com/openedx/enterprise-catalog.git" ) non_release_repos=( @@ -56,6 +57,7 @@ ssh_repos=( "git@github.com:openedx/cs_comments_service.git" "git@github.com:edx/ecommerce.git" "git@github.com:openedx/edx-notes-api.git" + "git@github.com:openedx/enterprise-catalog.git" "git@github.com:openedx/edx-platform.git" "git@github.com:openedx/xqueue.git" "git@github.com:edx/edx-analytics-dashboard.git"