diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml new file mode 100644 index 0000000000..7c772abdec --- /dev/null +++ b/.github/workflows/add-to-project.yml @@ -0,0 +1,16 @@ +name: Add new issues to appropriate GitHub Projects + +on: + issues: + types: + - opened + +jobs: + add_to_arch_bom_board: + uses: openedx/.github/.github/workflows/add-issue-to-a-project.yml@master + secrets: + GITHUB_APP_ID: ${{ secrets.GRAPHQL_AUTH_APP_ID }} + GITHUB_APP_PRIVATE_KEY: ${{ secrets.GRAPHQL_AUTH_APP_PEM }} + with: + ORGANIZATION: edx + PROJECT_NUMBER: 11 diff --git a/.github/workflows/cli-tests.yml b/.github/workflows/cli-tests.yml index b3258f63c0..292481e76a 100644 --- a/.github/workflows/cli-tests.yml +++ b/.github/workflows/cli-tests.yml @@ -31,7 +31,7 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: setup python uses: actions/setup-python@v4 with: @@ -48,15 +48,17 @@ jobs: sudo apt update sudo apt install docker-ce containerd.io docker version - docker-compose --version + docker compose --version # Note: we cannot use Docker Desktop because it has not been licensed for use in GithubActions - name: Docker installation - Mac if: ${{ matrix.os.name == 'mac' }} run: | - brew install lima docker + brew install lima docker docker-compose limactl start --name=default template://docker echo "DOCKER_HOST=unix:///Users/runner/.lima/default/sock/docker.sock" >> $GITHUB_ENV + mkdir -p ~/.docker/cli-plugins + ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose - name: Install Python dependencies run: make requirements diff --git a/.github/workflows/follow-up-devstack-bugs.yml b/.github/workflows/follow-up-devstack-bugs.yml index 5b4000d5ff..2f0bc366ea 100644 --- a/.github/workflows/follow-up-devstack-bugs.yml +++ b/.github/workflows/follow-up-devstack-bugs.yml @@ -11,7 +11,7 @@ jobs: issues: write steps: - name: Add comment - uses: peter-evans/create-or-update-comment@411d7f9b4092af4736447c5420752e3b2be55ec1 + uses: peter-evans/create-or-update-comment@46da6c0d98504aed6fc429519a258b951f23f474 with: issue-number: ${{ github.event.issue.number }} body: | diff --git a/.github/workflows/pr-automerge-open-release.yml b/.github/workflows/pr-automerge-open-release.yml deleted file mode 100644 index 25af91e052..0000000000 --- a/.github/workflows/pr-automerge-open-release.yml +++ /dev/null @@ -1,24 +0,0 @@ -# Enable automerging for named release branches. -# See the reusable workflow for details: -# https://github.com/openedx/.github/.github/workflows/pr-automerge-open-release.yml - -name: Automerge BTR open-release PRs - -on: - issue_comment: - branches: - - open-release/* - types: - - created - - edited - pull_request_target: - branches: - - open-release/* - types: - - opened - - edited - - ready_for_review - -jobs: - automerge: - uses: openedx/.github/.github/workflows/pr-automerge-open-release.yml@master diff --git a/.github/workflows/provisioning-tests.yml b/.github/workflows/provisioning-tests.yml index 050753fc3d..027b9ce755 100644 --- a/.github/workflows/provisioning-tests.yml +++ b/.github/workflows/provisioning-tests.yml @@ -5,9 +5,11 @@ name: Provisioning tests on: push: branches: [master] + paths-ignore: + - '**.rst' pull_request: - branches: - - '**' + paths-ignore: + - '**.rst' schedule: # run at 7:30 am M-F - cron: '30 11 * * 1-5' @@ -26,11 +28,11 @@ jobs: os: - ubuntu-20.04 # Ubuntu 20.04 "Focal Fossa" python-version: [ '3.8' ] - services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue] + services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue, analyticsapi+insights+lms] fail-fast: false # some services can be flaky; let others run to completion even if one fails steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: setup python uses: actions/setup-python@v4 with: @@ -46,10 +48,13 @@ jobs: sudo apt update sudo apt install docker-ce containerd.io docker version - docker-compose --version + docker compose --version - name: free up disk space - run: sudo apt remove --purge -y ghc-* azure-cli google-cloud-sdk hhvm llvm-* dotnet-* powershell mono-* php* ruby* + # 2023-09-28: google-cloud-sdk removed from this list because it was intermittently + # unavailable as an apt package to remove, and might be migrating to snap. If more + # disk space is needed, see if the snap is installed, and remove that. + run: sudo apt remove --purge -y ghc-* azure-cli hhvm llvm-* dotnet-* powershell mono-* php* ruby* - name: set up requirements run: make requirements @@ -65,11 +70,15 @@ jobs: - name: provision run: make dev.provision.${{matrix.services}} - - name: dev.up + - name: "Bring up services" run: make dev.up.${{matrix.services}} - - name: dev.check - run: make dev.check.${{matrix.services}} + - name: "Wait for services to become ready" + run: | + # Wait a reasonable amount of time for services to come up. If they + # don't, then call the checks one more time to ensure that diagnostic + # information is printed out. (It's suppressed by wait-for.) + timeout 5m make dev.wait-for.${{matrix.services}} || timeout 1m make dev.check.${{matrix.services}} - name: notify on failure if: ${{ failure() && github.ref == 'refs/heads/master' }} diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index ba78da34c7..1587f09e96 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: setup python uses: actions/setup-python@v4 with: diff --git a/Makefile b/Makefile index 194792cea1..629da474a3 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,8 @@ # make dev.attach.credentials # make dev.pull.registrar+cms # make dev.up.lms -# make dev.up.without-deps.lms+forum+discovery+mysql57+elasticsearch710+memcached -# make dev.restart-container.mysql57+lms +# make dev.up.without-deps.lms+forum+discovery+mysql80+elasticsearch710+memcached +# make dev.restart-container.mysql80+lms # There are also "prefix-form" targets, which are simply an alternate way to spell # the 'dev.' targets. @@ -189,7 +189,7 @@ dev.pull.without-deps.%: ## Pull latest Docker images for specific services. @scripts/send_metrics.py wrap "dev.pull.without-deps.$*" impl-dev.pull.without-deps.%: ## Pull latest Docker images for specific services. - docker-compose pull $$(echo $* | tr + " ") + docker compose pull $$(echo $* | tr + " ") dev.pull: @scripts/send_metrics.py wrap "$@" @@ -205,7 +205,7 @@ dev.pull.%: ## Pull latest Docker images for services and their dependencies. @scripts/send_metrics.py wrap "dev.pull.$*" impl-dev.pull.%: ## Pull latest Docker images for services and their dependencies. - docker-compose pull --include-deps $$(echo $* | tr + " ") + docker compose pull --include-deps $$(echo $* | tr + " ") ######################################################################################## # Developer interface: Database management. @@ -253,18 +253,18 @@ $(foreach db_service,$(DB_SERVICES_LIST),\ dev.migrate: | $(_db_migration_targets) ## Run migrations for applicable default services. dev.migrate.cms: - docker-compose exec cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-cms' + docker compose exec cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-cms' dev.migrate.lms: - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-lms' + docker compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-lms' dev.migrate.%: ## Run migrations on a service. - docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' + docker compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' dev.drop-db: _expects-database.dev.drop-db dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mysql container. - docker-compose exec -T mysql57 bash -c "mysql --execute=\"DROP DATABASE $*;\"" + docker compose exec -T mysql80 bash -c "mysql --execute=\"DROP DATABASE $*;\"" ######################################################################################## @@ -274,7 +274,7 @@ dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mys dev.up.attach: _expects-service.dev.up.attach impl-dev.up.attach.%: ## Bring up a service and its dependencies + and attach to it. - docker-compose up $* + docker compose up $* dev.up.attach.%: ## Bring up a service and its dependencies + and attach to it. @scripts/send_metrics.py wrap "dev.up.attach.$*" @@ -298,7 +298,7 @@ dev.up.with-watchers.%: ## Bring up services and their dependencies + asset watc dev.up.without-deps: _expects-service-list.dev.up.without-deps impl-dev.up.without-deps.%: dev.check-memory ## Bring up services by themselves. - docker-compose up -d --no-deps $$(echo $* | tr + " ") + docker compose up -d --no-deps $$(echo $* | tr + " ") dev.up.without-deps.%: ## Bring up services by themselves. @scripts/send_metrics.py wrap "dev.up.without-deps.$*" @@ -316,7 +316,7 @@ dev.up.large-and-slow: dev.up.$(DEFAULT_SERVICES) ## Bring up default services. @echo # at least one statement so that dev.up.% doesn't run too impl-dev.up.%: dev.check-memory ## Bring up services and their dependencies. - docker-compose up -d $$(echo $* | tr + " ") + docker compose up -d $$(echo $* | tr + " ") ifeq ($(ALWAYS_CACHE_PROGRAMS),true) make dev.cache-programs endif @@ -326,32 +326,32 @@ dev.up.%: @scripts/send_metrics.py wrap "dev.up.$*" dev.ps: ## View list of created services and their statuses. - docker-compose ps + docker compose ps dev.print-container.%: ## Get the ID of the running container for a given service. @# Can be run as ``make --silent --no-print-directory dev.print-container.$service`` for just ID. - @echo $$(docker-compose ps --quiet $*) + @echo $$(docker compose ps --quiet $*) dev.restart-container: ## Restart all service containers. - docker-compose restart $$(echo $* | tr + " ") + docker compose restart $$(echo $* | tr + " ") dev.restart-container.%: ## Restart specific services' containers. - docker-compose restart $$(echo $* | tr + " ") + docker compose restart $$(echo $* | tr + " ") dev.stop: ## Stop all running services. - docker-compose stop + docker compose stop dev.stop.%: ## Stop specific services. - docker-compose stop $$(echo $* | tr + " ") + docker compose stop $$(echo $* | tr + " ") dev.kill: ## Kill all running services. - docker-compose stop + docker compose stop dev.kill.%: ## Kill specific services. - docker-compose kill $$(echo $* | tr + " ") + docker compose kill $$(echo $* | tr + " ") dev.rm-stopped: ## Remove stopped containers. Does not affect running containers. - docker-compose rm --force + docker compose rm --force dev.down: ## Documentation for a change to naming @echo "dev.down has been renamed to dev.remove-containers. If this doesn't seem like what you were looking for, you probably want to be using dev.stop instead. See docs for more details." @@ -360,10 +360,10 @@ dev.down.%: @echo "dev.down has been renamed to dev.remove-containers. If this doesn't seem like what you were looking for, you probably want to be using dev.stop instead. See docs for more details." dev.remove-containers: ## Stop and remove containers and networks for all services. - docker-compose down + docker compose down dev.remove-containers.%: ## Stop and remove containers for specific services. - docker-compose rm --force --stop $$(echo $* | tr + " ") + docker compose rm --force --stop $$(echo $* | tr + " ") ######################################################################################## @@ -381,8 +381,11 @@ dev.check: dev.check.$(DEFAULT_SERVICES) ## Run checks for the default service s dev.check.%: # Run checks for a given service or set of services. $(WINPTY) bash ./check.sh $* +dev.wait-for.%: ## Wait for these services to become ready + $(WINPTY) bash ./wait-ready.sh $$(echo $* | tr + " ") + dev.validate: ## Print effective Docker Compose config, validating files in COMPOSE_FILE. - docker-compose config + docker compose config ######################################################################################## @@ -395,20 +398,20 @@ dev.cache-programs: ## Copy programs from Discovery to Memcached for use in LMS. dev.restart-devserver: _expects-service.dev.restart-devserver dev.restart-devserver.forum: - docker-compose exec -T forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' + docker compose exec -T forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' dev.forum.build-indices: ## Build indices for forum service - docker-compose exec -T forum bash -c "cd forum && source ruby_env && source devstack_forum_env && cd cs_comments_service/ && bin/rake search:rebuild_indices" + docker compose exec -T forum bash -c "cd forum && source ruby_env && source devstack_forum_env && cd cs_comments_service/ && bin/rake search:rebuild_indices" dev.restart-devserver.%: ## Kill an edX service's development server. Watcher should restart it. # Applicable to Django services only. - docker-compose exec -T $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + docker compose exec -T $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' dev.logs: ## View logs from running containers. - docker-compose logs -f + docker compose logs -f dev.logs.%: ## View the logs of the specified service container. - docker-compose logs -f --tail=500 $* + docker compose logs -f --tail=500 $* dev.attach: _expects-service.dev.attach @@ -418,54 +421,61 @@ dev.attach.%: ## Attach to the specified service container process for debugging dev.shell: _expects-service.dev.shell dev.shell.credentials: - docker-compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' + docker compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' dev.shell.discovery: - docker-compose exec discovery env TERM=$(TERM) bash -c '/bin/bash' + docker compose exec discovery env TERM=$(TERM) bash -c '/bin/bash' dev.shell.ecommerce: - docker-compose exec ecommerce env TERM=$(TERM) /bin/bash + docker compose exec ecommerce env TERM=$(TERM) /bin/bash dev.shell.registrar: - docker-compose exec registrar env TERM=$(TERM) /bin/bash + docker compose exec registrar env TERM=$(TERM) /bin/bash dev.shell.xqueue: - docker-compose exec xqueue env TERM=$(TERM) /bin/bash + docker compose exec xqueue env TERM=$(TERM) /bin/bash dev.shell.lms: - docker-compose exec lms env TERM=$(TERM) bash -c '/bin/bash' + docker compose exec lms env TERM=$(TERM) bash -c '/bin/bash' dev.shell.lms_watcher: - docker-compose exec lms_watcher env TERM=$(TERM) bash -c '/bin/bash' + docker compose exec lms_watcher env TERM=$(TERM) bash -c '/bin/bash' dev.shell.cms: - docker-compose exec cms env TERM=$(TERM) bash -c '/bin/bash' + docker compose exec cms env TERM=$(TERM) bash -c '/bin/bash' dev.shell.cms_watcher: - docker-compose exec cms_watcher env TERM=$(TERM) bash -c '/bin/bash' + docker compose exec cms_watcher env TERM=$(TERM) bash -c '/bin/bash' dev.shell.xqueue_consumer: - docker-compose exec xqueue_consumer env TERM=$(TERM) /bin/bash + docker compose exec xqueue_consumer env TERM=$(TERM) /bin/bash dev.shell.analyticsapi: docker exec -it edx.devstack.analyticsapi env TERM=$(TERM) bash -c '/bin/bash' dev.shell.insights: - docker-compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /bin/bash' + docker compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /bin/bash' dev.shell.%: ## Run a shell on the specified service's container. - docker-compose exec $* /bin/bash + docker compose exec $* /bin/bash dev.dbshell: - docker-compose exec mysql57 bash -c "mysql" + docker compose exec mysql80 bash -c "mysql" + +DB_NAMES_LIST = credentials discovery ecommerce notes registrar xqueue edxapp edxapp_csmh dashboard analytics-api reports reports_v1 +_db_copy8_targets = $(addprefix dev.dbcopy8.,$(DB_NAMES_LIST)) +dev.dbcopyall8: ## Copy data from old mysql 5.7 containers into new mysql8 dbs + $(MAKE) dev.up.mysql57+mysql80 + $(MAKE) dev.wait-for.mysql57+mysql80 + $(MAKE) $(_db_copy8_targets) dev.dbcopy8.%: ## Copy data from old mysql 5.7 container into a new 8 db - docker-compose exec mysql57 bash -c "mysqldump $*" > .dev/$*.sql - docker-compose exec -T mysql80 bash -c "mysql $*" < .dev/$*.sql + docker compose exec mysql57 mysqldump "$*" > .dev/$*.sql + docker compose exec -T mysql80 mysql "$*" < .dev/$*.sql rm .dev/$*.sql dev.dbshell.%: ## Run a SQL shell on the given database. - docker-compose exec mysql57 bash -c "mysql $*" + docker compose exec mysql80 bash -c "mysql $*" # List of Makefile targets to run static asset generation, in the form dev.static.$(service) # Services will only have their asset generation added here @@ -478,13 +488,13 @@ $(foreach asset_service,$(ASSET_SERVICES_LIST),\ dev.static: | $(_asset_compilation_targets) dev.static.lms: - docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms' + docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms' dev.static.cms: - docker-compose exec -T cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets cms' + docker compose exec -T cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets cms' dev.static.%: ## Rebuild static assets for the specified service's container. - docker-compose exec -T $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + docker compose exec -T $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' ######################################################################################## @@ -605,18 +615,18 @@ docs: ## generate Sphinx HTML documentation, including API docs validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container. touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile - docker-compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile + docker compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers. - @docker-compose logs chrome 2>&1 | grep "VNC password" | tail -1 - @docker-compose logs firefox 2>&1 | grep "VNC password" | tail -1 + @docker compose logs chrome 2>&1 | grep "VNC password" | tail -1 + @docker compose logs firefox 2>&1 | grep "VNC password" | tail -1 devpi-password: ## Get the root devpi password for the devpi container. - docker-compose exec devpi bash -c "cat /data/server/.serverpassword" + docker compose exec devpi bash -c "cat /data/server/.serverpassword" hadoop-application-logs-%: ## View hadoop logs by application Id. - docker-compose exec nodemanager yarn logs -applicationId $* + docker compose exec nodemanager yarn logs -applicationId $* create-test-course: ## Provisions cms, and ecommerce with course(s) in test-course.json. $(WINPTY) bash ./course-generator/create-courses.sh --cms --ecommerce course-generator/test-course.json diff --git a/check.sh b/check.sh index 0113cb6263..891cf9b10e 100755 --- a/check.sh +++ b/check.sh @@ -43,16 +43,39 @@ run_check() { local cmd="$3" echo "> $cmd" set +e # Disable exit-on-error - if $cmd; then # Run the command itself and check if it succeeded. + if bash -c "$cmd"; then # Run the command itself and check if it succeeded. succeeded="$succeeded $check_name" else - docker-compose logs "$service" + docker compose logs --tail 500 "$service" # Just show recent logs, not all history failed="$failed $check_name" fi set -e # Re-enable exit-on-error echo # Newline } +mysql_run_check() { + container_name="$1" + mysql_probe="SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" + run_check "${container_name}_query" "$container_name" \ + "docker compose exec -T $(printf %q "$container_name") mysql -uroot -se $(printf %q "$mysql_probe")" +} + +if should_check mysql57; then + echo "Checking MySQL 5.7 query endpoint:" + mysql_run_check mysql57 +fi + +if should_check mysql80; then + echo "Checking MySQL 8.0 query endpoint:" + mysql_run_check mysql80 +fi + +if should_check mongo; then + echo "Checking MongoDB status:" + run_check mongo_status mongo \ + "docker compose exec -T mongo mongo --eval \"db.serverStatus()\"" +fi + if should_check registrar; then echo "Checking Registrar heartbeat:" run_check registrar_heartbeat registrar \ @@ -64,15 +87,17 @@ if should_check lms; then run_check lms_heartbeat lms \ "curl --fail -L http://localhost:18000/heartbeat" - echo "Checking CMS heartbeat:" - run_check cms_heartbeat lms \ - "curl --fail -L http://localhost:18010/heartbeat" - echo "Validating LMS volume:" run_check lms_volume lms \ "make validate-lms-volume" fi +if should_check cms; then + echo "Checking CMS heartbeat:" + run_check cms_heartbeat cms \ + "curl --fail -L http://localhost:18010/heartbeat" +fi + if should_check ecommerce; then echo "Checking ecommerce health:" run_check ecommerce_heartbeat ecommerce \ diff --git a/configuration_files/ecommerce.yml b/configuration_files/ecommerce.yml index 1ec3646cb7..e36bac008c 100644 --- a/configuration_files/ecommerce.yml +++ b/configuration_files/ecommerce.yml @@ -25,7 +25,7 @@ DATABASES: ATOMIC_REQUESTS: true CONN_MAX_AGE: 60 ENGINE: django.db.backends.mysql - HOST: edx.devstack.mysql57 + HOST: edx.devstack.mysql80 NAME: ecommerce OPTIONS: connect_timeout: 10 diff --git a/configuration_files/xqueue.yml b/configuration_files/xqueue.yml index 9a7df3b41e..0a648a8b11 100644 --- a/configuration_files/xqueue.yml +++ b/configuration_files/xqueue.yml @@ -5,7 +5,7 @@ DATABASES: ATOMIC_REQUESTS: true CONN_MAX_AGE: 0 ENGINE: django.db.backends.mysql - HOST: edx.devstack.mysql57 + HOST: edx.devstack.mysql80 NAME: xqueue OPTIONS: {} PASSWORD: password diff --git a/course-generator/create-courses.sh b/course-generator/create-courses.sh index 861ce50d27..c4c714a5ec 100755 --- a/course-generator/create-courses.sh +++ b/course-generator/create-courses.sh @@ -7,14 +7,14 @@ echo "Parsing options" container_error=false for arg in "$@"; do if [ $arg == "--cms" ]; then - if [ ! "$(docker-compose exec lms bash -c 'echo "Course will be created for cms"; exit $?')" ]; then + if [ ! "$(docker compose exec lms bash -c 'echo "Course will be created for cms"; exit $?')" ]; then echo "Issue with cms container" container_error=true else cms=true fi elif [ $arg == "--ecommerce" ]; then - if [ ! "$(docker-compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then + if [ ! "$(docker compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then echo "Issue with ecommerce container" container_error=true else @@ -42,10 +42,10 @@ done < "${@: -1}" if $cms ; then echo "Creating courses on cms." - docker-compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'" + docker compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'" fi if $ecommerce ; then echo "Creating courses on ecommerce." - docker-compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" + docker compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" fi diff --git a/credentials/generate_program_certificate.sh b/credentials/generate_program_certificate.sh index 2347b03d03..0f1b0dda51 100755 --- a/credentials/generate_program_certificate.sh +++ b/credentials/generate_program_certificate.sh @@ -2,27 +2,27 @@ echo 'Attempting to award a program certificate to the edX user' echo 'Updating Discovery...' echo 'Adding assets to the edX demo organization' -docker-compose exec -T discovery bash -c 'mkdir /edx/app/discovery/discovery/provision-temp' +docker compose exec -T discovery bash -c 'mkdir /edx/app/discovery/discovery/provision-temp' docker cp ./assets edx.devstack.discovery:/edx/app/discovery/discovery/provision-temp/assets -docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py add_logos_to_organization --partner=edX --logo=/edx/app/discovery/discovery/provision-temp/assets/demo-asset-logo.png --certificate_logo=/edx/app/discovery/discovery/provision-temp/assets/demo-asset-certificate-logo.png --banner_image=/edx/app/discovery/discovery/provision-temp/assets/demo-asset-banner-image.png' -docker-compose exec -T discovery bash -c 'rm -rf /edx/app/discovery/discovery/provision-temp' +docker compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py add_logos_to_organization --partner=edX --logo=/edx/app/discovery/discovery/provision-temp/assets/demo-asset-logo.png --certificate_logo=/edx/app/discovery/discovery/provision-temp/assets/demo-asset-certificate-logo.png --banner_image=/edx/app/discovery/discovery/provision-temp/assets/demo-asset-banner-image.png' +docker compose exec -T discovery bash -c 'rm -rf /edx/app/discovery/discovery/provision-temp' echo 'Updating credentials...' echo 'setting catalog and lms base urls' -docker-compose exec -T credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py create_or_update_site --site-domain example.com --site-name example.com --platform-name edX --tos-url https://www.edx.org/edx-terms-service --privacy-policy-url https://www.edx.org/edx-privacy-policy --homepage-url https://www.edx.org --company-name "edX Inc." --certificate-help-url https://edx.readthedocs.org/projects/edx-guide-for-students/en/latest/SFD_certificates.html#web-certificates --lms-url-root http://edx.devstack.lms:18000/ --catalog-api-url http://edx.devstack.discovery:18381/api/v1/ --theme-name edx.org' +docker compose exec -T credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py create_or_update_site --site-domain example.com --site-name example.com --platform-name edX --tos-url https://www.edx.org/edx-terms-service --privacy-policy-url https://www.edx.org/edx-privacy-policy --homepage-url https://www.edx.org --company-name "edX Inc." --certificate-help-url https://edx.readthedocs.org/projects/edx-guide-for-students/en/latest/SFD_certificates.html#web-certificates --lms-url-root http://edx.devstack.lms:18000/ --catalog-api-url http://edx.devstack.discovery:18381/api/v1/ --theme-name edx.org' echo 'copying discovery catalog' -docker-compose exec -T credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py copy_catalog' +docker compose exec -T credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py copy_catalog' echo 'creating a program certificate configuration' -docker-compose exec -T credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py create_program_certificate_configuration' +docker compose exec -T credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py create_program_certificate_configuration' echo 'Updating LMS...' echo 'creating a credentials API connection' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms create_credentials_api_configuration' +docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms create_credentials_api_configuration' echo 'changing edX user enrollment in demo course from audit to verified' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms change_enrollment -u edx -c course-v1:edX+DemoX+Demo_Course --from audit --to verified' +docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms change_enrollment -u edx -c course-v1:edX+DemoX+Demo_Course --from audit --to verified' echo 'manually ID verifying edX user' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms manual_verifications --email edx@example.com' +docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms manual_verifications --email edx@example.com' echo 'generating course certificate' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms cert_generation -u 3 -c course-v1:edX+DemoX+Demo_Course' +docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms cert_generation -u 3 -c course-v1:edX+DemoX+Demo_Course' echo 'notifying credentials' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker notify_credentials --courses course-v1:edX+DemoX+Demo_Course --notify_programs' +docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker notify_credentials --courses course-v1:edX+DemoX+Demo_Course --notify_programs' diff --git a/destroy.sh b/destroy.sh index 1c92e23784..235367f7bd 100755 --- a/destroy.sh +++ b/destroy.sh @@ -4,5 +4,5 @@ set -eu -o pipefail read -p "This will delete all data in your devstack. Would you like to proceed? [y/n] " -r if [[ $REPLY =~ ^[Yy]$ ]] then - docker-compose down -v + docker compose down -v fi diff --git a/docker-compose.yml b/docker-compose.yml index 5fe5c4d61f..f3abc79f04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -239,7 +239,7 @@ services: redis: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.redis" hostname: redis.devstack.edx - image: redis:6.2.12 + image: redis:7.2 command: redis-server --requirepass password networks: default: @@ -293,13 +293,14 @@ services: depends_on: - lms - memcached + - mysql80 - mysql57 # Allows attachment to the credentials service using 'docker attach '. stdin_open: true tty: true environment: CACHE_LOCATION: edx.devstack.memcached:11211 - DB_HOST: edx.devstack.mysql57 + DB_HOST: edx.devstack.mysql80 SOCIAL_AUTH_EDX_OIDC_URL_ROOT: http://edx.devstack.lms:18000/oauth2 ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 @@ -318,6 +319,7 @@ services: depends_on: - elasticsearch710 - memcached + - mysql80 - mysql57 - opensearch12 - redis @@ -327,7 +329,7 @@ services: environment: # This next DB_MIGRATION_HOST line can be removed once edx/configuration has been updated with this value for # a while and most people have had a chance to do a "make pull" to get the latest images. - DB_MIGRATION_HOST: edx.devstack.mysql57 + DB_MIGRATION_HOST: edx.devstack.mysql80 TEST_ELASTICSEARCH_URL: "edx.devstack.elasticsearch710" ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 @@ -350,6 +352,7 @@ services: - discovery - lms - memcached + - mysql80 - mysql57 # Allows attachment to the ecommerce service using 'docker attach '. stdin_open: true @@ -377,6 +380,7 @@ services: - devpi - elasticsearch710 - lms + - mysql80 - mysql57 image: openedx/edx-notes-api-dev:${OPENEDX_RELEASE:-latest} networks: @@ -387,7 +391,7 @@ services: - "18120:18120" environment: DB_ENGINE: "django.db.backends.mysql" - DB_HOST: "edx.devstack.mysql57" + DB_HOST: "edx.devstack.mysql80" DB_NAME: "notes" DB_PASSWORD: "password" DB_PORT: "3306" @@ -427,6 +431,7 @@ services: - forum - memcached - mongo + - mysql80 - mysql57 # Allows attachment to the LMS service using 'docker attach '. stdin_open: true @@ -463,6 +468,7 @@ services: hostname: insights.devstack.edx depends_on: - analyticsapi + - mysql80 - mysql57 - lms - memcached @@ -470,7 +476,7 @@ services: stdin_open: true tty: true environment: - DB_HOST: edx.devstack.mysql57 + DB_HOST: edx.devstack.mysql80 DB_NAME: dashboard DB_PORT: 3306 DB_USER: analytics001 @@ -495,13 +501,14 @@ services: container_name: edx.devstack.analyticsapi hostname: analyticsapi depends_on: + - mysql80 - mysql57 - elasticsearch710 command: bash -c 'source /edx/app/analytics_api/analytics_api_env && while true; do python /edx/app/analytics_api/analytics_api/manage.py runserver 0.0.0.0:19001 --settings analyticsdataserver.settings.devstack; sleep 2; done' stdin_open: true tty: true environment: - DB_HOST: edx.devstack.mysql57 + DB_HOST: edx.devstack.mysql80 DB_PORT: 3306 DB_USER: analytics001 DB_PASSWORD: password @@ -520,6 +527,7 @@ services: depends_on: - discovery - lms + - mysql80 - mysql57 - memcached - redis @@ -528,7 +536,7 @@ services: stdin_open: true tty: true environment: - DB_HOST: edx.devstack.mysql57 + DB_HOST: edx.devstack.mysql80 DB_NAME: registrar DB_PORT: 3306 DB_USER: registrar001 @@ -562,12 +570,13 @@ services: hostname: registrar-worker.devstack.edx depends_on: - lms + - mysql80 - mysql57 - redis stdin_open: true tty: true environment: - DB_HOST: edx.devstack.mysql57 + DB_HOST: edx.devstack.mysql80 DB_NAME: registrar DB_PORT: 3306 DB_USER: registrar001 @@ -600,6 +609,7 @@ services: - lms - memcached - mongo + - mysql80 - mysql57 # Allows attachment to the CMS service using 'docker attach '. stdin_open: true @@ -639,6 +649,7 @@ services: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue - ${PWD}/configuration_files/xqueue.yml:/edx/etc/xqueue.yml depends_on: + - mysql80 - mysql57 environment: XQUEUE_CFG: "/edx/etc/xqueue.yml" @@ -658,6 +669,7 @@ services: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue - ${PWD}/configuration_files/xqueue.yml:/edx/etc/xqueue.yml depends_on: + - mysql80 - mysql57 networks: default: diff --git a/docs/developing_on_named_release_branches.rst b/docs/developing_on_named_release_branches.rst index b48a8f25da..4dbd5950c1 100644 --- a/docs/developing_on_named_release_branches.rst +++ b/docs/developing_on_named_release_branches.rst @@ -13,10 +13,19 @@ By default, the startup steps in :doc:`getting_started` will install the devstac checkout of each service repository #. Continue with step 3 in :doc:`getting_started` to pull the correct docker images. -All ``make`` target and ``docker-compose`` calls should now use the correct +All ``make`` target and ``docker compose`` calls should now use the correct images until you change or unset ``OPENEDX_RELEASE`` again. To work on the master branches and ``latest`` images, unset ``OPENEDX_RELEASE`` or set it to an empty string. +Note that older versions of devstack may have different prerequisites. In particular, +releases before Quince will need support for the ``docker-compose`` syntax as +well as the newer ``docker compose``. The easiest way to do this is to add +is to add a shell script with the following and put it on the PATH under the name docker-compose: + + .. code:: sh + + #!/bin/bash + docker compose "$@" How do I run multiple named Open edX releases on same machine? -------------------------------------------------------------- diff --git a/docs/devstack_faq.rst b/docs/devstack_faq.rst index 13ecd9fd9c..0e0161dd94 100644 --- a/docs/devstack_faq.rst +++ b/docs/devstack_faq.rst @@ -54,7 +54,7 @@ starts, you have a few options: automatically on a regular basis. See `building images for devstack`_ for more information. * You can update your requirements files as appropriate and then build your own updated image for the service as described above, tagging it such that - ``docker-compose`` will use it instead of the last image you downloaded. + ``docker compose`` will use it instead of the last image you downloaded. (Alternatively, you can temporarily edit ``docker-compose.yml`` to replace the ``image`` entry for that service with the ID of your new image.) You should be sure to modify the variable override for the version of the @@ -134,7 +134,7 @@ How do I connect to the databases from an outside editor? --------------------------------------------------------- To connect to the databases from an outside editor (such as MySQLWorkbench), -first uncomment these lines from ``docker-compose.yml``'s ``mysql57`` section +first uncomment these lines from ``docker-compose.yml``'s ``mysql80`` section .. code:: yaml @@ -145,8 +145,8 @@ Then bring your mysql container down and back up by running: .. code:: sh - docker-compose stop mysql57 - docker-compose up -d mysql57 + docker compose stop mysql80 + docker compose up -d mysql80 Then connect using the values below. Note that the username and password will vary depending on the database. For all of the options, see ``provision.sql``. @@ -158,7 +158,7 @@ vary depending on the database. For all of the options, see ``provision.sql``. If you have trouble connecting, ensure the port was mapped successfully by running ``make dev.ps`` and looking for a line like this: -``edx.devstack.mysql57 docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``. +``edx.devstack.mysql80 docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``. How do I build the service images myself? ----------------------------------------- @@ -209,7 +209,7 @@ To access the MySQL shell for a particular database, run: .. code:: sh - make dev.shell.mysql57 + make dev.shell.mysql80 mysql use ; diff --git a/docs/devstack_interface.rst b/docs/devstack_interface.rst index 5aa38fe860..6a24046aef 100644 --- a/docs/devstack_interface.rst +++ b/docs/devstack_interface.rst @@ -1,7 +1,7 @@ Devstack Interface ------------------ -Devstack comes built in with many useful make commands that act as an user interface. This UI is essentially a make wrapper around docker-compose commands. We attempt to give a short summary of what the make commands do below, but it would be a good idea for you to familiarize yourself with some knowledge of docker-compose. +Devstack comes built in with many useful make commands that act as an user interface. This UI is essentially a make wrapper around ``docker compose`` commands. We attempt to give a short summary of what the make commands do below, but it would be a good idea for you to familiarize yourself with some knowledge of docker compose. Due to the organic nature of how this user interface came into being, there are often multiple ways to do the same thing. The two main variants of commands are of the form: ``dev.ACTION.SERVICE`` vs ``SERVICE-ACTION``. The ``SERVICE-ACTION`` variant tends to be more tab-completion friendly. @@ -18,7 +18,9 @@ Examples: make dev.logs.frontend-app-gradebook make frontend-app-gradebook-logs -The user interface for devstack often also gives you both big hammers(``make dev.pull.large-and-slow``) and small hammers(``make dev.pull.``) to do things. It is recommend you opt for the small hammer commands, because they often tend to be alot faster. +The user interface for devstack often also gives you both big hammers (``make dev.pull.large-and-slow``) and small hammers (``make dev.pull.``) to do things. It is recommend you opt for the small hammer commands, because they often tend to be a lot faster. + +See the ``make help`` section below for a complete list of commands. Useful Commands and Summary ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -27,7 +29,7 @@ Useful Commands and Summary - ``dev.pull.`` - Pull latest Docker images for the service and its dependencies - When to use: If you have not used Devstack for a while or pulled new images for a while, the installed requirements in your containers might no longer match those used by your code. So it is recommended you pull new images whenever you want your containers to have the latest requirements(python libraries...) installed. + When to use: If you have not used Devstack for a while or pulled new images for a while, the installed requirements in your containers might no longer match those used by your code. So it is recommended you pull new images whenever you want your containers to have the latest requirements (python libraries...) installed. Note: for new service images to be used, you first need to bring down those services and then bring them back up after a pull. @@ -116,7 +118,7 @@ Useful Commands and Summary Variation: ``make -restart-devserver``. -- ``dev.restart-container.`` restarts service container. This is essentially a stronger version of ``dev.restrart-devserver`` +- ``dev.restart-container.`` restarts service container. This is essentially a stronger version of ``dev.restart-devserver`` Note: this will only restart and not its dependencies @@ -129,3 +131,93 @@ Useful Commands and Summary - ``dev.restore`` will restore your database volumes to the backups created using ``dev.backup`` Warning: This will overwrite your databases. Only use if you want all your database volumes to revert back to the backup. + +Make Help +~~~~~~~~~ + +The following ``make help`` output was generated in 09-2023 to make these commands searchable in documentation. + +If you want to ensure you are getting the latest listing, simply use ``make help``. + +.. code:: sh + + % make help + Please use `make ' where is one of + build-courses Build course and provision cms, and ecommerce with it. + create-test-course Provisions cms, and ecommerce with course(s) in test-course.json. + dev.attach.% Attach to the specified service container process for debugging & seeing logs. + dev.backup Write all data volumes to the host. + dev.cache-programs Copy programs from Discovery to Memcached for use in LMS. + dev.check Run checks for the default service set. + dev.check-memory Check if enough memory has been allocated to Docker. + dev.checkout Check out "openedx-release/$OPENEDX_RELEASE" in each repo if set, use default branch otherwise. + dev.clone Clone service repos to the parent directory. + dev.clone.https Clone service repos using HTTPS method to the parent directory. + dev.clone.ssh Clone service repos using SSH method to the parent directory. + dev.dbcopy8.% Copy data from old mysql 5.7 container into a new 8 db + dev.dbshell.% Run a SQL shell on the given database. + dev.destroy Irreversibly remove all devstack-related containers and networks (though not data volumes) + dev.destroy.coursegraph Remove all coursegraph data. + dev.down Documentation for a change to naming + dev.drop-db.% Irreversably drop the contents of a MySQL database in each mysql container. + dev.forum.build-indices Build indices for forum service + dev.kill Kill all running services. + dev.kill.% Kill specific services. + dev.logs View logs from running containers. + dev.logs.% View the logs of the specified service container. + dev.migrate Run migrations for applicable default services. + dev.migrate.% Run migrations on a service. + dev.print-container.% Get the ID of the running container for a given service. + dev.provision Provision dev environment with default services, and then stop them. + dev.provision.% Provision specified services. + dev.prune Prune dangling docker images, containers, and networks. Useful when you get the 'no space left on device' error + dev.ps View list of created services and their statuses. + dev.pull.% Pull latest Docker images for services and their dependencies. + dev.pull.large-and-slow Pull latest Docker images required by default services. + dev.pull.without-deps.% Pull latest Docker images for specific services. + dev.remove-containers Stop and remove containers and networks for all services. + dev.remove-containers.% Stop and remove containers for specific services. + dev.reset Attempt to reset the local devstack to the default branch working state without destroying data. + dev.reset-repos Attempt to reset the local repo checkouts to the default branch working state. + dev.restart-container Restart all service containers. + dev.restart-container.% Restart specific services' containers. + dev.restart-devserver.% Kill an edX service's development server. Watcher should restart it. + dev.restore Restore all data volumes from the host. WILL OVERWRITE ALL EXISTING DATA! + dev.rm-stopped Remove stopped containers. Does not affect running containers. + dev.shell.% Run a shell on the specified service's container. + dev.static.% Rebuild static assets for the specified service's container. + dev.stats Get per-container CPU and memory utilization data. + dev.status Prints the status of all git repositories. + dev.stop Stop all running services. + dev.stop.% Stop specific services. + dev.up.attach.% Bring up a service and its dependencies + and attach to it. + dev.up.large-and-slow Bring up default services. + dev.up.shell.% Bring up a service and its dependencies + shell into it. + dev.up.with-programs Bring up default services + cache programs in LMS. + dev.up.with-programs.% Bring up services and their dependencies + cache programs in LMS. + dev.up.with-watchers Bring up default services + asset watcher containers. + dev.up.with-watchers.% Bring up services and their dependencies + asset watcher containers. + dev.up.without-deps.% Bring up services by themselves. + dev.up.without-deps.shell.% Bring up a service by itself + shell into it. + dev.validate Print effective Docker Compose config, validating files in COMPOSE_FILE. + dev.wait-for.% Wait for these services to become ready + devpi-password Get the root devpi password for the devpi container. + docs generate Sphinx HTML documentation, including API docs + hadoop-application-logs-% View hadoop logs by application Id. + help Display this help message. + impl-dev.clone.https Clone service repos using HTTPS method to the parent directory. + impl-dev.clone.ssh Clone service repos using SSH method to the parent directory. + impl-dev.provision Provision dev environment with default services, and then stop them. + impl-dev.provision.% Provision specified services. + impl-dev.pull.% Pull latest Docker images for services and their dependencies. + impl-dev.pull.without-deps.% Pull latest Docker images for specific services. + impl-dev.up.% Bring up services and their dependencies. + impl-dev.up.attach.% Bring up a service and its dependencies + and attach to it. + impl-dev.up.without-deps.% Bring up services by themselves. + metrics-opt-in To opt into basic data collection to help improve devstack + metrics-opt-out To opt out of metrics data collection + requirements install development environment requirements + selfcheck Check that the Makefile is free of Make syntax errors. + upgrade Upgrade requirements with pip-tools. + validate-lms-volume Validate that changes to the local workspace are reflected in the LMS container. + vnc-passwords Get the VNC passwords for the Chrome and Firefox Selenium containers. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index b34251c9c1..c8697ac930 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -8,21 +8,21 @@ You will need to have the following installed: - make - Python 3.8 -- Docker, including ``docker-compose`` +- Docker, including ``docker compose`` -This project requires **Docker 17.06+ CE**. We recommend Docker Stable, but +This project requires **Docker 19.03+ CE**. We recommend Docker Stable, but Docker Edge should work as well. Ensure that your Docker installation includes -``docker-compose``; on some operating systems (e.g. Ubuntu Linux) this may require +``docker compose``; on some operating systems (e.g. Ubuntu Linux) this may require a separate package. **NOTE:** Switching between Docker Stable and Docker Edge will remove all images and settings. Don't forget to restore your memory setting and be prepared to provision. -For macOS users, please use `Docker for Mac`_. Previous Mac-based tools (e.g. -boot2docker) are *not* supported. Please be aware that the `licensing terms`_ for -Docker for Mac (aka Docker Desktop) may mean that it is no longer -free for your organization's use. +For macOS users, please use `Docker for Mac`_, which comes with ``docker +compose``. Previous Mac-based tools (e.g. boot2docker) are *not* supported. +Please be aware that the `licensing terms`_ for Docker for Mac (aka Docker +Desktop) may mean that it is no longer free for your organization's use. Since a Docker-based devstack runs many containers, you should configure Docker with a sufficient @@ -32,9 +32,14 @@ does work. `Docker for Windows`_ may work but has not been tested and is *not* supported. -If you are using Linux, use the ``overlay2`` storage driver, kernel version -4.0+ and *not* ``overlay``. To check which storage driver your -``docker-daemon`` uses, run the following command. +If you are using Linux, developers on Ubuntu (and Debian) should ensure +they've uninstalled docker.io and docker-compose from the main Ubuntu +repositories and instead install docker-ce and docker-compose-plugin from the +official Docker package repository: +https://docs.docker.com/engine/install/ubuntu/. Also they should use the +``overlay2`` storage driver, kernel version 4.0+ and *not* ``overlay``. To +check which storage driver your ``docker-daemon`` uses, run the following +command. .. code:: sh diff --git a/docs/index.rst b/docs/index.rst index c2c1e8810d..0f4b9e6934 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,4 +26,5 @@ Contents pycharm_integration testing_and_debugging troubleshoot_general_tips + manual_upgrades advanced_configuration diff --git a/docs/manual_upgrades.rst b/docs/manual_upgrades.rst new file mode 100644 index 0000000000..888b4640b6 --- /dev/null +++ b/docs/manual_upgrades.rst @@ -0,0 +1,55 @@ +Manual upgrade instructions +########################### + +Occasionally there is a change to devstack that requires existing devstack installations to be manually upgraded. When this happens, instructions should be added here. + +Please add new instructions to the top, include a date, and make a post in the `Devstack forum `_. + +(If you just need to update your devstack to the latest version of everything, see :doc:`updating_devstack`.) + +2023-10-05 - MySQL upgrade from version 5.7 to 8.0 +************************************************** + +The MySQL service has been upgraded from version 5.7 to 8.0. Developers will need to follow the following instructions. + +1. Stop the running containers :: + + make dev.stop + +2. Take latest ``git pull`` of ``devstack`` and ``edx-platform`` +3. Take the latest pull of images :: + + make dev.pull + +4. Run provisioning command :: + + make dev.provision + +5. [Optional] Additionally, there is a database copy command to help you transfer data from MySQL 5.7 to 8.0. After provisioning use the following command :: + + make dev.dbcopyall8 + +This command copies the following databases: + +- credentials +- discovery +- ecommerce +- registrar +- notes +- edxapp +- xqueue +- edxapp_csmh +- dashboard +- analytics-api +- reports +- reports_v1 + +If you prefer not to copy all databases, update ``DB_NAMES_LIST`` in the ``Makefile`` of devstack before running the dbcopy command. + +2023-08-02 - Forum upgrade from Ruby 2 to 3 +******************************************* + +The forum service has been upgraded from Ruby 2 to Ruby 3. Developers who use forum will need to pull the new image and reprovision the service:: + + make dev.pull.forum # pull in new forum image + make dev.provision.forum # provision forum service diff --git a/docs/troubleshoot_general_tips.rst b/docs/troubleshoot_general_tips.rst index 17435e0cef..0a0b3761e2 100644 --- a/docs/troubleshoot_general_tips.rst +++ b/docs/troubleshoot_general_tips.rst @@ -134,7 +134,7 @@ for example: The most common culprit is an infinite restart loop where an error during service startup causes the process to exit, but we've configured -``docker-compose`` to immediately try starting it again (so the container will +``docker compose`` to immediately try starting it again (so the container will stay running long enough for you to use a shell to investigate and fix the problem). Make sure the set of packages installed in the container matches what your current code branch expects; you may need to rerun ``pip`` on a @@ -273,6 +273,17 @@ During ``make requirements`` there would be an error:: This was resolved in July 2023 with https://github.com/openedx/edx-platform/pull/32732. +Cannot run ``make upgrade`` in lms shell due to missing wget +------------------------------------------------------------ + +``make upgrade`` or ``make compile-requirements`` in lms-shell would produce an error about wget:: + + wget -O "requirements/common_constraints.txt" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + /bin/sh: 1: wget: not found + make[1]: *** [Makefile:115: requirements/common_constraints.txt] Error 127 + +This error was `introduced `_ and `resolved `_ in September 2023. While this can be solved by updating your devstack, you can also run ``apt update; apt install wget`` from lms-shell to resolve this temporarily. + .. _update your repos and pull the latest images: Updating Devstack @@ -291,6 +302,11 @@ Like with pulling images, you can also narrow these commands to specific service Running ``make dev.reset`` will do all the above for all services, which can be useful but takes much more time. It will also run a full ``docker system prune -f`` to get rid of unused images and networks. +Manual Upgrades +=============== + +Sometimes there is a change to devstack that requires existing devstack installations to be manually upgraded. See :doc:`manual_upgrades` for recent cases of this. + Starting From Scratch ===================== diff --git a/enterprise/provision.sh b/enterprise/provision.sh index 18b8947a44..dcc5c08b96 100755 --- a/enterprise/provision.sh +++ b/enterprise/provision.sh @@ -2,7 +2,7 @@ set -eu -o pipefail set -x -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user enterprise_worker enterprise_worker@example.com --staff' -cat enterprise/worker_permissions.py | docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user enterprise_worker enterprise_worker@example.com --staff' +cat enterprise/worker_permissions.py | docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "enterprise-backend-service-key" --client-secret "enterprise-backend-service-secret" enterprise-backend-service enterprise_worker' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "enterprise-backend-service-key" --client-secret "enterprise-backend-service-secret" enterprise-backend-service enterprise_worker' diff --git a/load-db.sh b/load-db.sh index 99be594989..3863639ef1 100755 --- a/load-db.sh +++ b/load-db.sh @@ -16,6 +16,6 @@ then fi echo "Loading the $1 database..." -mysql_container=$(make --silent --no-print-directory dev.print-container.mysql57) +mysql_container=$(make --silent --no-print-directory dev.print-container.mysql80) docker exec -i "$mysql_container" mysql -uroot $1 < $1.sql echo "Finished loading the $1 database!" diff --git a/options.mk b/options.mk index 1537641b67..bde253e627 100644 --- a/options.mk +++ b/options.mk @@ -90,4 +90,4 @@ credentials+discovery+ecommerce+insights+lms+registrar+cms # All third-party services. # Separated by plus signs. Listed in alphabetical order for clarity. THIRD_PARTY_SERVICES ?= \ -chrome+coursegraph+devpi+elasticsearch710+firefox+memcached+mongo+mysql57+opensearch12+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica +chrome+coursegraph+devpi+elasticsearch710+firefox+memcached+mongo+mysql57+mysql80+opensearch12+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica diff --git a/programs/provision.sh b/programs/provision.sh index 32697100d7..ca7cd50d19 100755 --- a/programs/provision.sh +++ b/programs/provision.sh @@ -45,7 +45,7 @@ docker_exec() { /edx/app/$app/$repo/manage.py $cmd " - docker-compose exec -T "$service" bash -e -c "$CMDS" + docker compose exec -T "$service" bash -e -c "$CMDS" } provision_ida() { diff --git a/provision-analyticsapi.sh b/provision-analyticsapi.sh index 0e4fffc373..3d564ad629 100755 --- a/provision-analyticsapi.sh +++ b/provision-analyticsapi.sh @@ -7,16 +7,16 @@ set -x name=analyticsapi port=19001 -docker-compose up -d ${name} +docker compose up -d ${name} echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make develop' -- ${name} +docker compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make develop' -- ${name} echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && export DJANGO_SETTINGS_MODULE="analyticsdataserver.settings.devstack" && cd /edx/app/analytics_api/analytics_api && make migrate-all' -- ${name} +docker compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && export DJANGO_SETTINGS_MODULE="analyticsdataserver.settings.devstack" && cd /edx/app/analytics_api/analytics_api && make migrate-all' -- ${name} echo -e "${GREEN}Creating default user and authentication token for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && python manage.py set_api_key edx edx' -- ${name} +docker compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && python manage.py set_api_key edx edx' -- ${name} echo -e "${GREEN}Loading test data for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make loaddata' -- ${name} +docker compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make loaddata' -- ${name} diff --git a/provision-coursegraph.sh b/provision-coursegraph.sh index 4bbc5d0602..337c99c234 100755 --- a/provision-coursegraph.sh +++ b/provision-coursegraph.sh @@ -9,14 +9,14 @@ set -x # Also, this gives us a chance to refresh the container in case it's gotten into # a weird state. echo -e "${GREEN} Ensuring Coursegraph image is up to date...${NC}" -docker-compose rm --force --stop coursegraph -docker-compose pull coursegraph +docker compose rm --force --stop coursegraph +docker compose pull coursegraph echo -e "${GREEN} Starting Coursegraph and CMS...${NC}" -docker-compose up -d coursegraph cms +docker compose up -d coursegraph cms sleep 10 # Give Neo4j some time to boot up. echo -e "${GREEN} Updating CMS courses in Coursegraph...${NC}" -docker-compose exec cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && ./manage.py cms dump_to_neo4j --host coursegraph.devstack.edx --user neo4j --password edx' +docker compose exec cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && ./manage.py cms dump_to_neo4j --host coursegraph.devstack.edx --user neo4j --password edx' echo -e "${GREEN} Coursegraph is now up-to-date with CMS!${NC}" diff --git a/provision-credentials.sh b/provision-credentials.sh index 77fd1ce02e..f73da9a6eb 100755 --- a/provision-credentials.sh +++ b/provision-credentials.sh @@ -10,26 +10,26 @@ set -x name=credentials port=18150 -docker-compose up -d $name +docker compose up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements' -- "$name" +docker compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" +docker compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" +docker compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" echo -e "${GREEN}Configuring site for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' +docker compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker compose exec -T ${name} bash -e -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" # Restart credentials devserver. make dev.restart-devserver.credentials diff --git a/provision-discovery.sh b/provision-discovery.sh index 04f959d619..8b947a47c9 100755 --- a/provision-discovery.sh +++ b/provision-discovery.sh @@ -3,24 +3,24 @@ set -eu -o pipefail set -x -docker-compose up -d lms -docker-compose up -d cms -docker-compose up -d ecommerce +docker compose up -d lms +docker compose up -d cms +docker compose up -d ecommerce sleep 5 # Give above services some time to boot up ./provision-ida.sh discovery discovery 18381 -docker-compose exec -T discovery bash -e -c 'rm -rf /edx/var/discovery/*' -docker-compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --lms-coursemode-api-url "http://edx.devstack.lms:18000/api/course_modes/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/" --studio-url "http://edx.devstack.cms:18010/" --publisher-url "http://edx.devstack.frontend-app-publisher:18400/"' +docker compose exec -T discovery bash -e -c 'rm -rf /edx/var/discovery/*' +docker compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --lms-coursemode-api-url "http://edx.devstack.lms:18000/api/course_modes/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/" --studio-url "http://edx.devstack.cms:18010/" --publisher-url "http://edx.devstack.frontend-app-publisher:18400/"' set +e # FIXME[bash-e]: Bash scripts should use -e -- but this script fails # (after many retries) when trying to talk to ecommerce -docker-compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' -docker-compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py add_provisioning_data' +docker compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' +docker compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py add_provisioning_data' set -e -docker-compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' +docker compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' # Add demo program ./programs/provision.sh discovery diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index 85cda52306..13a6e094dc 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -8,6 +8,6 @@ set -x ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --enable-microfrontend-for-basket-page=1 --payment-microfrontend-url=http://localhost:1998' -docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' -docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' +docker compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --enable-microfrontend-for-basket-page=1 --payment-microfrontend-url=http://localhost:1998' +docker compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' +docker compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' diff --git a/provision-forum.sh b/provision-forum.sh index 342aa49875..98a8104eed 100755 --- a/provision-forum.sh +++ b/provision-forum.sh @@ -2,5 +2,5 @@ set -eu -o pipefail set -x -docker-compose up -d forum -docker-compose exec -T forum bash -e -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/ && bin/rake search:initialize' +docker compose up -d forum +docker compose exec -T forum bash -e -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/ && bin/rake search:initialize' diff --git a/provision-ida-user.sh b/provision-ida-user.sh index ccc2ecf0dc..df68df6fce 100755 --- a/provision-ida-user.sh +++ b/provision-ida-user.sh @@ -13,8 +13,8 @@ client_port=$3 echo -e "${GREEN}Creating service user and OAuth2 applications for ${app_name}...${NC}" # Create the service user. -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" # Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" diff --git a/provision-ida.sh b/provision-ida.sh index 483b54ce3f..2236c65d75 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -12,17 +12,17 @@ container_name=${4:-$1} # (Optional) The name of the container. If missing, wil make dev.up.$app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" -docker-compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" +docker compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" echo -e "${GREEN}Running migrations for ${app_name}...${NC}" -docker-compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" +docker compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" echo -e "${GREEN}Creating super-user for ${app_name}...${NC}" -docker-compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" +docker compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" ./provision-ida-user.sh $app_name $client_name $client_port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${app_name}...${NC}" -docker-compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" +docker compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" diff --git a/provision-insights.sh b/provision-insights.sh index 864eb0a85e..17959eae1c 100755 --- a/provision-insights.sh +++ b/provision-insights.sh @@ -7,20 +7,20 @@ set -x name=insights port=18110 -docker-compose up -d insights +docker compose up -d insights echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights && make develop' -- ${name} +docker compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights && make develop' -- ${name} # # Install Insights npm dependencies -docker-compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights/ && npm ci && ./npm-post-install.sh' +docker compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights/ && npm ci && ./npm-post-install.sh' echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && export DJANGO_SETTINGS_MODULE="analytics_dashboard.settings.devstack" && cd /edx/app/insights/insights && make migrate' -- ${name} +docker compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && export DJANGO_SETTINGS_MODULE="analytics_dashboard.settings.devstack" && cd /edx/app/insights/insights && make migrate' -- ${name} ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights && make static' -- "$name" +docker compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights && make static' -- "$name" diff --git a/provision-lms.sh b/provision-lms.sh index ffd48b9e14..78247d1147 100755 --- a/provision-lms.sh +++ b/provision-lms.sh @@ -12,47 +12,47 @@ cms_port=18010 # Bring edxapp containers online for app in "${apps[@]}"; do - docker-compose up -d $app + docker compose up -d $app done # install git for both LMS and CMS for app in "${apps[@]}"; do - docker-compose exec -T $app bash -e -c 'apt-get update && apt-get -y install --no-install-recommends git' + docker compose exec -T $app bash -e -c 'apt-get update && apt-get -y install --no-install-recommends git' - docker-compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' + docker compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process - docker-compose restart $app + docker compose restart $app done # Run edxapp migrations first since they are needed for the service users and OAuth clients # Make migrate runs migrations for both lms and cms. -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && make migrate' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && make migrate' -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py lms showmigrations --database student_module_history --traceback --pythonpath=. --settings devstack_docker' -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py lms migrate --database student_module_history --noinput --traceback --pythonpath=. --settings devstack_docker' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py lms showmigrations --database student_module_history --traceback --pythonpath=. --settings devstack_docker' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py lms migrate --database student_module_history --noinput --traceback --pythonpath=. --settings devstack_docker' -docker-compose exec -T cms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py cms showmigrations --database student_module_history --traceback --pythonpath=. --settings devstack_docker' -docker-compose exec -T cms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py cms migrate --database student_module_history --noinput --traceback --pythonpath=. --settings devstack_docker' +docker compose exec -T cms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py cms showmigrations --database student_module_history --traceback --pythonpath=. --settings devstack_docker' +docker compose exec -T cms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py cms migrate --database student_module_history --noinput --traceback --pythonpath=. --settings devstack_docker' # Create a superuser for edxapp -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' # Create an enterprise service user for edxapp and give them appropriate permissions ./enterprise/provision.sh # Enable the LMS-E-Commerce integration -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' # Create demo course and users -#docker-compose exec -T lms bash -e -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' +#docker compose exec -T lms bash -e -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' if [[ ${DEVSTACK_SKIP_DEMO-false} == "true" ]] then echo "Skipping import of demo course. DEVSTACK_SKIP_DEMO is set to true" else - docker-compose exec -T lms bash -e -c 'git clone https://github.com/openedx/edx-demo-course.git /tmp/edx-demo-course' - docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /edx/var/edxapp/data /tmp/edx-demo-course && rm -rf /tmp/edx-demo-course' + docker compose exec -T lms bash -e -c 'git clone https://github.com/openedx/edx-demo-course.git /tmp/edx-demo-course' + docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /edx/var/edxapp/data /tmp/edx-demo-course && rm -rf /tmp/edx-demo-course' fi demo_hashed_password='pbkdf2_sha256$20000$TjE34FJjc3vv$0B7GUmH8RwrOc/BvMoxjb5j8EgnWTt3sxorDANeF7Qw=' @@ -60,24 +60,24 @@ for user in honor audit verified staff ; do email="$user@example.com" # Set staff flag for staff user if [[ $user == "staff" ]] ; then - docker-compose exec -T lms bash -e -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker --service-variant lms manage_user $user $email --initial-password-hash '$demo_hashed_password' --staff" + docker compose exec -T lms bash -e -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker --service-variant lms manage_user $user $email --initial-password-hash '$demo_hashed_password' --staff" else - docker-compose exec -T lms bash -e -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker --service-variant lms manage_user $user $email --initial-password-hash '$demo_hashed_password'" + docker compose exec -T lms bash -e -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker --service-variant lms manage_user $user $email --initial-password-hash '$demo_hashed_password'" fi if [[ "${DEVSTACK_SKIP_DEMO-false}" != "true" ]] then # Enroll users in the demo course - docker-compose exec -T lms bash -e -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker --service-variant lms enroll_user_in_course -e $email -c course-v1:edX+DemoX+Demo_Course" + docker compose exec -T lms bash -e -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker --service-variant lms enroll_user_in_course -e $email -c course-v1:edX+DemoX+Demo_Course" fi done # Fix missing vendor file by clearing the cache -docker-compose exec -T lms bash -e -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' +docker compose exec -T lms bash -e -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' # Create static assets for both LMS and CMS for app in "${apps[@]}"; do - docker-compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' + docker compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' done # Allow LMS SSO for CMS diff --git a/provision-notes.sh b/provision-notes.sh index 81075a6499..8c824858db 100755 --- a/provision-notes.sh +++ b/provision-notes.sh @@ -9,24 +9,24 @@ name=edx_notes_api port=18734 client_name=edx-notes # The name of the Oauth client stored in the edxapp DB. -docker-compose up -d $name +docker compose up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/notes && make requirements' -- "$name" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/notes && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/notes && make migrate' -- "$name" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/notes && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cho "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/notes/manage.py shell' -- "$name" +docker compose exec -T ${name} bash -e -c 'cho "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/notes/manage.py shell' -- "$name" ./provision-ida-user.sh $name $client_name $port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/notes && make static' -- "$name" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/notes && make static' -- "$name" # This will build the elasticsearch index for notes. echo -e "${GREEN}Creating indexes for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/notes/ && python manage.py search_index --rebuild -f' +docker compose exec -T ${name} bash -e -c 'cd /edx/app/notes/ && python manage.py search_index --rebuild -f' diff --git a/provision-registrar.sh b/provision-registrar.sh index 4d695c747b..1e527eff3f 100755 --- a/provision-registrar.sh +++ b/provision-registrar.sh @@ -7,20 +7,20 @@ set -x name=registrar port=18734 -docker-compose up -d $name +docker compose up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/registrar && make requirements' -- "$name" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/registrar && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/registrar && make migrate' -- "$name" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/registrar && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/registrar && make createsuperuser' -- "$name" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/registrar && make createsuperuser' -- "$name" ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c ' if ! cd /edx/app/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker compose exec -T ${name} bash -e -c ' if ! cd /edx/app/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-retirement-user.sh b/provision-retirement-user.sh index 2df6d814ee..812d92b1c5 100755 --- a/provision-retirement-user.sh +++ b/provision-retirement-user.sh @@ -9,5 +9,5 @@ app_name=$1 user_name=$2 echo -e "${GREEN}Creating retirement service user ${user_name} and DOT Application ${app_name}...${NC}" -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" diff --git a/provision-xqueue.sh b/provision-xqueue.sh index 8fde5dafce..d983cc57c2 100755 --- a/provision-xqueue.sh +++ b/provision-xqueue.sh @@ -3,11 +3,11 @@ set -eu -o pipefail set -x # Bring up XQueue, we don't need the consumer for provisioning -docker-compose up -d xqueue +docker compose up -d xqueue # Update dependencies -docker-compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && make requirements' +docker compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && make requirements' # Run migrations -docker-compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py migrate' +docker compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py migrate' # Add users that graders use to fetch data, there's one default user in Ansible which is part of our settings -docker-compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py update_users' +docker compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py update_users' diff --git a/provision.sh b/provision.sh index 9e63f23c02..258abf6e7a 100755 --- a/provision.sh +++ b/provision.sh @@ -122,70 +122,48 @@ fi echo -e "${GREEN}Will provision the following:\n ${to_provision_ordered}${NC}" # Bring the databases online. -docker-compose up -d mysql57 -docker-compose up -d mysql80 +docker compose up -d mysql57 +docker compose up -d mysql80 if needs_mongo "$to_provision_ordered"; then - docker-compose up -d mongo + docker compose up -d mongo fi # Ensure the MySQL5 server is online and usable echo "${GREEN}Waiting for MySQL 5.7.${NC}" -until docker-compose exec -T mysql57 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null -do - printf "." - sleep 1 -done +make dev.wait-for.mysql57 # Ensure the MySQL8 server is online and usable echo "${GREEN}Waiting for MySQL 8.0.${NC}" -until docker-compose exec -T mysql80 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null -do - printf "." - sleep 1 -done +make dev.wait-for.mysql80 # In the event of a fresh MySQL container, wait a few seconds for the server to restart # See https://github.com/docker-library/mysql/issues/245 for why this is necessary. sleep 10 echo "${GREEN}Waiting for MySQL 5.7 to restart.${NC}" -until docker-compose exec -T mysql57 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null -do - printf "." - sleep 1 -done - +make dev.wait-for.mysql57 echo -e "${GREEN}MySQL5 ready.${NC}" echo "${GREEN}Waiting for MySQL 8.0 to restart.${NC}" -until docker-compose exec -T mysql80 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null -do - printf "." - sleep 1 -done - +make dev.wait-for.mysql80 echo -e "${GREEN}MySQL8 ready.${NC}" # Ensure that the MySQL databases and users are created for all IDAs. # (A no-op for databases and users that already exist). echo -e "${GREEN}Ensuring MySQL 5.7 databases and users exist...${NC}" -docker-compose exec -T mysql57 bash -e -c "mysql -uroot mysql" < provision.sql +docker compose exec -T mysql57 bash -e -c "mysql -uroot mysql" < provision.sql echo -e "${GREEN}Ensuring MySQL 8.0 databases and users exist...${NC}" -docker-compose exec -T mysql80 bash -e -c "mysql -uroot mysql" < provision-mysql80.sql +docker compose exec -T mysql80 bash -e -c "mysql -uroot mysql" < provision-mysql80.sql # If necessary, ensure the MongoDB server is online and usable # and create its users. if needs_mongo "$to_provision_ordered"; then echo -e "${GREEN}Waiting for MongoDB...${NC}" # mongo container and mongo process/shell inside the container - until docker-compose exec -T mongo mongo --eval "db.serverStatus()" &> /dev/null - do - printf "." - sleep 1 - done + make dev.wait-for.mongo echo -e "${GREEN}MongoDB ready.${NC}" echo -e "${GREEN}Creating MongoDB users...${NC}" - docker-compose exec -T mongo bash -e -c "mongo" < mongo-provision.js + docker compose exec -T mongo bash -e -c "mongo" < mongo-provision.js else echo -e "${GREEN}MongoDB preparation not required; skipping.${NC}" fi diff --git a/requirements/base.in b/requirements/base.in index 8fce5cc217..1b9e6a1a3c 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,4 +1,4 @@ -c constraints.txt -docker-compose # For launching the devstack Docker containers -PyYAML # For parsing configuration files while generating offline installers +# Support for Apple Silicon begins with 6.0.0 +PyYAML>=6.0.0 # For parsing configuration files while generating offline installers diff --git a/requirements/base.txt b/requirements/base.txt index 5f746bdb4d..a0fda06220 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,69 +4,5 @@ # # make upgrade # -attrs==23.1.0 - # via jsonschema -bcrypt==4.0.1 - # via paramiko -certifi==2023.7.22 - # via requests -cffi==1.15.1 - # via - # cryptography - # pynacl -charset-normalizer==3.2.0 - # via requests -cryptography==41.0.3 - # via paramiko -distro==1.8.0 - # via docker-compose -docker[ssh]==6.1.3 - # via docker-compose -docker-compose==1.29.2 +pyyaml==6.0.1 # via -r requirements/base.in -dockerpty==0.4.1 - # via docker-compose -docopt==0.6.2 - # via docker-compose -idna==3.4 - # via requests -jsonschema==3.2.0 - # via docker-compose -packaging==23.1 - # via docker -paramiko==3.3.1 - # via docker -pycparser==2.21 - # via cffi -pynacl==1.5.0 - # via paramiko -pyrsistent==0.19.3 - # via jsonschema -python-dotenv==0.21.1 - # via docker-compose -pyyaml==5.4.1 - # via - # -r requirements/base.in - # docker-compose -requests==2.31.0 - # via - # docker - # docker-compose -six==1.16.0 - # via - # dockerpty - # jsonschema - # websocket-client -texttable==1.6.7 - # via docker-compose -urllib3==2.0.4 - # via - # docker - # requests -websocket-client==0.59.0 - # via - # docker - # docker-compose - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/dev.txt b/requirements/dev.txt index ffffb62039..7d6e5ab75c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,71 +4,16 @@ # # make upgrade # -attrs==23.1.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # jsonschema -bcrypt==4.0.1 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # paramiko -build==0.10.0 +build==1.0.0 # via # -r requirements/pip-tools.txt # pip-tools -certifi==2023.7.22 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # requests -cffi==1.15.1 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # cryptography - # pynacl -charset-normalizer==3.2.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # requests click==8.1.7 # via # -r requirements/pip-tools.txt # pip-tools -cryptography==41.0.3 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # paramiko distlib==0.3.7 # via virtualenv -distro==1.8.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose -docker[ssh]==6.1.3 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose -docker-compose==1.29.2 - # via - # -r requirements/base.txt - # -r requirements/test.txt -dockerpty==0.4.1 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose -docopt==0.6.2 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose exceptiongroup==1.1.3 # via # -r requirements/test.txt @@ -77,34 +22,21 @@ filelock==3.12.3 # via # tox # virtualenv -idna==3.4 +importlib-metadata==6.8.0 # via - # -r requirements/base.txt - # -r requirements/test.txt - # requests + # -r requirements/pip-tools.txt + # build iniconfig==2.0.0 # via # -r requirements/test.txt # pytest -jsonschema==3.2.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose packaging==23.1 # via - # -r requirements/base.txt # -r requirements/pip-tools.txt # -r requirements/test.txt # build - # docker # pytest # tox -paramiko==3.3.1 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker pexpect==4.8.0 # via -r requirements/test.txt pip-tools==7.3.0 @@ -122,56 +54,18 @@ ptyprocess==0.7.0 # pexpect py==1.11.0 # via tox -pycparser==2.21 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # cffi -pynacl==1.5.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # paramiko pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pyrsistent==0.19.3 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # jsonschema -pytest==7.4.0 +pytest==7.4.1 # via -r requirements/test.txt -python-dotenv==0.21.1 +pyyaml==6.0.1 # via # -r requirements/base.txt # -r requirements/test.txt - # docker-compose -pyyaml==5.4.1 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose -requests==2.31.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker - # docker-compose six==1.16.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # dockerpty - # jsonschema - # tox - # websocket-client -texttable==1.6.7 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose + # via tox tomli==2.0.1 # via # -r requirements/pip-tools.txt @@ -190,24 +84,16 @@ tox-battery==0.6.2 # via -r requirements/dev.in typing-extensions==4.7.1 # via filelock -urllib3==2.0.4 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker - # requests -virtualenv==20.24.3 +virtualenv==20.24.4 # via tox -websocket-client==0.59.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker - # docker-compose wheel==0.41.2 # via # -r requirements/pip-tools.txt # pip-tools +zipp==3.16.2 + # via + # -r requirements/pip-tools.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/doc.txt b/requirements/doc.txt index b66cd7a292..eb0767effb 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -8,59 +8,20 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx -attrs==23.1.0 - # via - # -r requirements/base.txt - # jsonschema babel==2.12.1 # via # pydata-sphinx-theme # sphinx -bcrypt==4.0.1 - # via - # -r requirements/base.txt - # paramiko beautifulsoup4==4.12.2 # via pydata-sphinx-theme bleach==6.0.0 # via readme-renderer certifi==2023.7.22 - # via - # -r requirements/base.txt - # requests -cffi==1.15.1 - # via - # -r requirements/base.txt - # cryptography - # pynacl + # via requests charset-normalizer==3.2.0 - # via - # -r requirements/base.txt - # requests -cryptography==41.0.3 - # via - # -r requirements/base.txt - # paramiko -distro==1.8.0 - # via - # -r requirements/base.txt - # docker-compose + # via requests doc8==1.1.1 # via -r requirements/doc.in -docker[ssh]==6.1.3 - # via - # -r requirements/base.txt - # docker-compose -docker-compose==1.29.2 - # via -r requirements/base.txt -dockerpty==0.4.1 - # via - # -r requirements/base.txt - # docker-compose -docopt==0.6.2 - # via - # -r requirements/base.txt - # docker-compose docutils==0.19 # via # doc8 @@ -69,37 +30,21 @@ docutils==0.19 # restructuredtext-lint # sphinx idna==3.4 - # via - # -r requirements/base.txt - # requests + # via requests imagesize==1.4.1 # via sphinx importlib-metadata==6.8.0 # via sphinx jinja2==3.1.2 # via sphinx -jsonschema==3.2.0 - # via - # -r requirements/base.txt - # docker-compose markupsafe==2.1.3 # via jinja2 packaging==23.1 # via - # -r requirements/base.txt - # docker # pydata-sphinx-theme # sphinx -paramiko==3.3.1 - # via - # -r requirements/base.txt - # docker pbr==5.11.1 # via stevedore -pycparser==2.21 - # via - # -r requirements/base.txt - # cffi pydata-sphinx-theme==0.13.3 # via sphinx-book-theme pygments==2.16.1 @@ -109,44 +54,21 @@ pygments==2.16.1 # pydata-sphinx-theme # readme-renderer # sphinx -pynacl==1.5.0 - # via - # -r requirements/base.txt - # paramiko -pyrsistent==0.19.3 - # via - # -r requirements/base.txt - # jsonschema -python-dotenv==0.21.1 - # via - # -r requirements/base.txt - # docker-compose -pytz==2023.3 +pytz==2023.3.post1 # via babel -pyyaml==5.4.1 - # via - # -r requirements/base.txt - # docker-compose +pyyaml==6.0.1 + # via -r requirements/base.txt readme-renderer==41.0 # via -r requirements/doc.in requests==2.31.0 - # via - # -r requirements/base.txt - # docker - # docker-compose - # sphinx + # via sphinx restructuredtext-lint==1.4.0 # via doc8 six==1.16.0 - # via - # -r requirements/base.txt - # bleach - # dockerpty - # jsonschema - # websocket-client + # via bleach snowballstemmer==2.2.0 # via sphinx -soupsieve==2.4.1 +soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 # via @@ -169,28 +91,13 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx stevedore==5.1.0 # via doc8 -texttable==1.6.7 - # via - # -r requirements/base.txt - # docker-compose tomli==2.0.1 # via doc8 typing-extensions==4.7.1 # via pydata-sphinx-theme urllib3==2.0.4 - # via - # -r requirements/base.txt - # docker - # requests + # via requests webencodings==0.5.1 # via bleach -websocket-client==0.59.0 - # via - # -r requirements/base.txt - # docker - # docker-compose zipp==3.16.2 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 80a54ce05a..135c9d9a31 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,10 +4,12 @@ # # make upgrade # -build==0.10.0 +build==1.0.0 # via pip-tools click==8.1.7 # via pip-tools +importlib-metadata==6.8.0 + # via build packaging==23.1 # via build pip-tools==7.3.0 @@ -18,8 +20,11 @@ tomli==2.0.1 # via # build # pip-tools + # pyproject-hooks wheel==0.41.2 # via pip-tools +zipp==3.16.2 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/test.txt b/requirements/test.txt index 51bf145a05..dc33211a25 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,125 +4,21 @@ # # make upgrade # -attrs==23.1.0 - # via - # -r requirements/base.txt - # jsonschema -bcrypt==4.0.1 - # via - # -r requirements/base.txt - # paramiko -certifi==2023.7.22 - # via - # -r requirements/base.txt - # requests -cffi==1.15.1 - # via - # -r requirements/base.txt - # cryptography - # pynacl -charset-normalizer==3.2.0 - # via - # -r requirements/base.txt - # requests -cryptography==41.0.3 - # via - # -r requirements/base.txt - # paramiko -distro==1.8.0 - # via - # -r requirements/base.txt - # docker-compose -docker[ssh]==6.1.3 - # via - # -r requirements/base.txt - # docker-compose -docker-compose==1.29.2 - # via -r requirements/base.txt -dockerpty==0.4.1 - # via - # -r requirements/base.txt - # docker-compose -docopt==0.6.2 - # via - # -r requirements/base.txt - # docker-compose exceptiongroup==1.1.3 # via pytest -idna==3.4 - # via - # -r requirements/base.txt - # requests iniconfig==2.0.0 # via pytest -jsonschema==3.2.0 - # via - # -r requirements/base.txt - # docker-compose packaging==23.1 - # via - # -r requirements/base.txt - # docker - # pytest -paramiko==3.3.1 - # via - # -r requirements/base.txt - # docker + # via pytest pexpect==4.8.0 # via -r requirements/test.in pluggy==1.3.0 # via pytest ptyprocess==0.7.0 # via pexpect -pycparser==2.21 - # via - # -r requirements/base.txt - # cffi -pynacl==1.5.0 - # via - # -r requirements/base.txt - # paramiko -pyrsistent==0.19.3 - # via - # -r requirements/base.txt - # jsonschema -pytest==7.4.0 +pytest==7.4.1 # via -r requirements/test.in -python-dotenv==0.21.1 - # via - # -r requirements/base.txt - # docker-compose -pyyaml==5.4.1 - # via - # -r requirements/base.txt - # docker-compose -requests==2.31.0 - # via - # -r requirements/base.txt - # docker - # docker-compose -six==1.16.0 - # via - # -r requirements/base.txt - # dockerpty - # jsonschema - # websocket-client -texttable==1.6.7 - # via - # -r requirements/base.txt - # docker-compose +pyyaml==6.0.1 + # via -r requirements/base.txt tomli==2.0.1 # via pytest -urllib3==2.0.4 - # via - # -r requirements/base.txt - # docker - # requests -websocket-client==0.59.0 - # via - # -r requirements/base.txt - # docker - # docker-compose - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/scripts/make_warn_default_large.sh b/scripts/make_warn_default_large.sh index cae0602296..d6d7115fca 100755 --- a/scripts/make_warn_default_large.sh +++ b/scripts/make_warn_default_large.sh @@ -31,7 +31,7 @@ You may prefer to use something like "make $target.lms" to target a smaller set of services. Learn more about the commands you can run at: - https://github.com/openedx/devstack/blob/master/docs/devstack_interface.rst + https://edx.readthedocs.io/projects/open-edx-devstack/en/latest/devstack_interface.html Without an explicit list of services, many devstack Make targets pull down Docker images you don't need or take up extra memory and CPU. You diff --git a/tests/warn_default.py b/tests/warn_default.py index c02a275a95..7645cd22f5 100644 --- a/tests/warn_default.py +++ b/tests/warn_default.py @@ -14,7 +14,7 @@ def test_warn_default(): p.expect(r'Are you sure you want to run this command') p.sendline('') - p.expect(r'Pulling lms') + p.expect(r'docker compose pull --include-deps') # Send ^C, don't wait for it to finish p.sendintr() diff --git a/update-dbs-init-sql-scripts.sh b/update-dbs-init-sql-scripts.sh index 6d16df0dfc..6591ba24bd 100755 --- a/update-dbs-init-sql-scripts.sh +++ b/update-dbs-init-sql-scripts.sh @@ -24,7 +24,7 @@ make dev.pull.lms+ecommerce make dev.provision.services.lms+ecommerce # dump schema and data from mysql databases in the mysql docker container and copy them to current directory in docker host -MYSQL_DOCKER_CONTAINER="$(make --silent --no-print-directory dev.print-container.mysql57)" +MYSQL_DOCKER_CONTAINER="$(make --silent --no-print-directory dev.print-container.mysql80)" for DB_NAME in "${DBS[@]}"; do DB_CREATION_SQL_SCRIPT="${DB_NAME}.sql" if [[ " ${EDXAPP_DBS[@]} " =~ " ${DB_NAME} " ]]; then diff --git a/wait-ready.sh b/wait-ready.sh new file mode 100755 index 0000000000..7fb1736685 --- /dev/null +++ b/wait-ready.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Wait for the listed services to become ready. +# +# This does not start the containers; that should be performed separately +# via `make dev.up` in order to allow for parallel startup. + +set -eu -o pipefail + +function print_usage { + echo "Usage: $0 service1 service2 ..." +} + +if [[ $# == 0 ]]; then + print_usage + exit 0 +fi + +for service_name in "$@"; do + until ./check.sh "$service_name" >/dev/null 2>&1; do + printf "." >&2 + sleep 1 + done + echo >&2 "$service_name is ready" +done