From 994e4b3215cb0cb18a0db2c37900c728739573ca Mon Sep 17 00:00:00 2001 From: LaRita Robinson Date: Fri, 8 Sep 2023 13:21:44 -0400 Subject: [PATCH 1/8] Remove reliance on DEPENDENCIES_NEXT --- Dockerfile | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 79598bd4..5d947869 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,15 +56,21 @@ ENV PATH="${PATH}:/app/fits" COPY --chown=1001:101 $APP_PATH/Gemfile* /app/samvera/hyrax-webapp/ RUN bundle install --jobs "$(nproc)" -COPY --chown=1001:101 $APP_PATH/Gemfile /app/samvera/hyrax-webapp/Gemfile_next -RUN DEPENDENCIES_NEXT=1 bundle install --jobs "$(nproc)" +# NOTE Bootboot enablement +# COPY --chown=1001:101 $APP_PATH/Gemfile /app/samvera/hyrax-webapp/Gemfile_next +# RUN DEPENDENCIES_NEXT=1 bundle install --jobs "$(nproc)" + +COPY --chown=1001:101 $APP_PATH/Gemfile /app/samvera/hyrax-webapp/Gemfile +RUN bundle install --jobs "$(nproc)" COPY --chown=1001:101 $APP_PATH /app/samvera/hyrax-webapp ARG SETTINGS__BULKRAX__ENABLED="false" -RUN sh -l -c " \ - DEPENDENCIES_NEXT=1 yarn install && \ - SOLR_URL=localhost DEPENDENCIES_NEXT=1 RAILS_ENV=production SECRET_KEY_BASE=fake-key-for-asset-building-only DB_ADAPTER=nulldb bundle exec rake assets:precompile" + +# NOTE Bootboot enablement +# RUN sh -l -c " \ +# DEPENDENCIES_NEXT=1 yarn install && \ +# SOLR_URL=localhost DEPENDENCIES_NEXT=1 RAILS_ENV=production SECRET_KEY_BASE=fake-key-for-asset-building-only DB_ADAPTER=nulldb bundle exec rake assets:precompile" RUN sh -l -c " \ yarn install && \ From 0c63cb310933d0478a3230fe76c3c75d8b7d4fd1 Mon Sep 17 00:00:00 2001 From: LaRita Robinson Date: Fri, 8 Sep 2023 13:34:07 -0400 Subject: [PATCH 2/8] Comment out yarn install Yarn is currently failing. --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5d947869..c0394509 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,9 +72,9 @@ ARG SETTINGS__BULKRAX__ENABLED="false" # DEPENDENCIES_NEXT=1 yarn install && \ # SOLR_URL=localhost DEPENDENCIES_NEXT=1 RAILS_ENV=production SECRET_KEY_BASE=fake-key-for-asset-building-only DB_ADAPTER=nulldb bundle exec rake assets:precompile" -RUN sh -l -c " \ - yarn install && \ - RAILS_ENV=production SECRET_KEY_BASE=fake-key-for-asset-building-only DB_ADAPTER=nulldb bundle exec rake assets:precompile" +# RUN sh -l -c " \ +# yarn install && \ +# RAILS_ENV=production SECRET_KEY_BASE=fake-key-for-asset-building-only DB_ADAPTER=nulldb bundle exec rake assets:precompile" CMD ./bin/web From 033627505ea1dbe9493eecf4bbe691514d75c3c1 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Fri, 8 Sep 2023 15:49:37 -0400 Subject: [PATCH 3/8] Removes unused AWS CodeDeploy config in appspec.yml --- appspec.yml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 appspec.yml diff --git a/appspec.yml b/appspec.yml deleted file mode 100644 index 23799108..00000000 --- a/appspec.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: 0.0 -os: linux -files: - - source: . - destination: /var/www/ams -hooks: - AfterInstall: - - location: bin/deploy/install.sh - timeout: 900 - runas: ec2-user - ValidateService: - - location: bin/deploy/verify_service.sh - timeout: 300 - runas: ec2-user From 7a30d79b3ced6a31e32c8165bc9509f02159bcce Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Fri, 8 Sep 2023 16:00:04 -0400 Subject: [PATCH 4/8] Removes rake task and script no longer used by AWS CodeDeploy --- bin/deploy/verify_service.sh | 6 ------ lib/tasks/verify_service.rake | 21 --------------------- 2 files changed, 27 deletions(-) delete mode 100755 bin/deploy/verify_service.sh delete mode 100644 lib/tasks/verify_service.rake diff --git a/bin/deploy/verify_service.sh b/bin/deploy/verify_service.sh deleted file mode 100755 index 3b32445f..00000000 --- a/bin/deploy/verify_service.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -le -cd /var/www/ams -bin/rails verify_service:sidekiq -bin/rails verify_service:homepage -echo "End of verify_service.sh" -exit 0; diff --git a/lib/tasks/verify_service.rake b/lib/tasks/verify_service.rake deleted file mode 100644 index e154b8aa..00000000 --- a/lib/tasks/verify_service.rake +++ /dev/null @@ -1,21 +0,0 @@ -require 'ams/sidekiq_process_manager' - -# Rake tasks to verify services are running. -# The script at bin/deploy/verify_service.sh is executed as part the -# ValidateService hook provided by CodeDeploy (see appspec.yml in project root). -# These rake tasks are used by that script to verify that services are running. -# Using ruby makes is more organized and easier to read. -namespace :verify_service do - task :sidekiq do - active_pids = AMS::SidekiqProcessManager.new.active_pids - raise 'Sidekiq is not running' unless active_pids.count > 0 - puts "#{active_pids.count} Sidekiq process running." - end - - task :homepage do - result = `curl -s -o /dev/null -I -w "%{http_code}" 127.0.0.1` - msg = "Request to 127.0.0.1 returned: #{result}" - raise msg unless result == '200' - puts msg - end -end From 50ea883c784a91f1186b7b3e0d2ad0a04b7b8eb8 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Fri, 8 Sep 2023 16:03:09 -0400 Subject: [PATCH 5/8] Removes install script no longer used by AWS CodeDeploy --- bin/deploy/install.sh | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100755 bin/deploy/install.sh diff --git a/bin/deploy/install.sh b/bin/deploy/install.sh deleted file mode 100755 index d4e84aac..00000000 --- a/bin/deploy/install.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -le -sudo yum -y update -source /etc/profile -echo "Running Deployment for ID: $DEPLOYMENT_ID" -export APP_HOME=/var/www/ams -sudo chown -R ec2-user:ec2-user $APP_HOME -cd $APP_HOME -echo "Ensuring bundler is installed..." -gem install bundler -bundle install --deployment --without development test - -# Print out versions of things for debugging -echo "ruby version:`ruby -v`" -echo "rails versions:`bundle exec rails -v`" -echo "node version:`node -v`" -if [ -z $SECRET_KEY_BASE ]; then - sudo echo "export SECRET_KEY_BASE=`bin/rails secret`" >> ~/.bashrc - source ~/.bashrc -fi -bin/rails db:migrate -bin/rails assets:precompile - -bin/rails g deployment_info_page --deployment_id $DEPLOYMENT_ID -sudo systemctl restart httpd -sleep 2 -SIDEKIQ_RESET=true bundle exec ruby bin/scripts/ensure_sidekiq_running.rb -sleep 2 -echo "End of install.sh" -exit 0; From 721a39693d50ee4dec7cfba7139e64608b8a11ec Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Fri, 8 Sep 2023 16:04:57 -0400 Subject: [PATCH 6/8] Removes code for checking/managing sidekiq processes that is no longer used --- bin/scripts/ensure_sidekiq_running.rb | 21 ---- lib/ams/sidekiq_process_manager.rb | 171 -------------------------- 2 files changed, 192 deletions(-) delete mode 100644 bin/scripts/ensure_sidekiq_running.rb delete mode 100644 lib/ams/sidekiq_process_manager.rb diff --git a/bin/scripts/ensure_sidekiq_running.rb b/bin/scripts/ensure_sidekiq_running.rb deleted file mode 100644 index 0ece2c79..00000000 --- a/bin/scripts/ensure_sidekiq_running.rb +++ /dev/null @@ -1,21 +0,0 @@ -require_relative '../../lib/ams/sidekiq_process_manager' - -sidekiq = AMS::SidekiqProcessManager.new -puts "#{sidekiq.active_pids.count} Sidekiq processes running." - -# If the reset flag was set, turn Sidekiq off first, and make sure no -# processes are running before continuing. -if ENV.fetch('SIDEKIQ_RESET', false) - # Here we specify a long wait time of 5 min. This should be at least as much - # as the timeout specified in Sidekiq config. - sidekiq.turn_off wait: 300 - if sidekiq.running? - puts "Could not reset Sidekiq processes. You may need to increase the " \ - "timeout in the Sidekiq config file, check logs for errors during " \ - "shutdown, and/or manually restart Sidekiq." - exit 1 - end -end - - -sidekiq.ensure_on(count: 1) diff --git a/lib/ams/sidekiq_process_manager.rb b/lib/ams/sidekiq_process_manager.rb deleted file mode 100644 index e607d915..00000000 --- a/lib/ams/sidekiq_process_manager.rb +++ /dev/null @@ -1,171 +0,0 @@ -require 'sidekiq/api' -require 'open3' -require 'logger' - -module AMS - class SidekiqProcessManager - - def running?; active_pids.count > 0; end - - def turn_on(count: 1, wait: 45) - count = [count.to_i, 0].max - logger.info "Turning on #{count} new Sidekiq processes; #{active_pids.count} currently running..." - new_total = count + active_pids.count - - # Start the specified number of new sidekiq processes. This will spawn new - # processes to run in the background. - start_sidekiq_processes(count: count) - - # Wait for all the processes to start. - try_until(wait) { active_pids.count == new_total } - - # Log a message telling whether all processes have started, or whether - # some are still trying to start, or may have errored. - if active_pids.count < new_total - logger.warn "Waited #{wait} seconds for #{new_total} processes " \ - "to be running, but #{active_pids.count} are currently " \ - "running. You may need to just wait longer, or check " \ - "#{sidekiq_log_file} for possible errors during startup." - else - logger.info "#{active_pids.count} Sidekiq processes running." - end - end - - def turn_off(count: active_pids.count, wait: 30) - count = [count.to_i, 0].max - # grab a subset of active pids to turn off. - pids = active_pids.sample(count) - logger.info "Turning off #{pids.count} of #{active_pids.count} Sidekiq processes..." - stop_processes(pids: pids) - - # Wait for processes to actually stop. NOTE: sending the TERM signal will - # cause the process to wait for workers to finish. The amount of time it - # will wait is set by the `timeout` configuration option when the process - # is started. This can be changed in a config file (e.g. - # config/sidekiq.yml) or by passing the -t option during startup. After - # waiting, if the workers for the process still have not finished, the - # process will be forcefully stopped with kill -9. - # Here we wait up to 5 min for the processes to turn off. - remaining = nil - try_until(wait) do - remaining = active_pids & pids - remaining.empty? - end - - if !remaining.empty? - logger.warn "Waited #{wait} seconds to stop #{pids.count} Sidekiq " \ - "process(es), but #{remaining.count} of those are still " \ - "running. Processes may finish shutting down shortly, or " \ - "there may have been an error during shutdown. See " \ - "#{sidekiq_log_file} for more details." - else - logger.info "#{pids.count} Sidekiq processes turned off." - end - end - - def ensure_on(count: 1) - count = [count.to_i, 0].max - logger.info "Ensuring #{count} Sidekiq process(es) running; found #{active_pids.count}." - if active_pids.count < count - turn_on(count: count - active_pids.count) - elsif active_pids.count > count - turn_off(count: active_pids.count - count) - end - end - - def active_pids - # NOTE: This command has been tested on OSX and Amazon linux. - `ps aux | egrep 'sidekiq.*\[[0-9]+ of [0-9]+ busy\]' | awk '{print $2}'`.split(/\s/) - end - - def logger=(logger) - raise ArgumentError, "Logger object expected but #{logger.class} was given" unless logger.is_a? Logger - @logger = logger - end - - def logger - @logger ||= Logger.new(STDOUT) - end - - private - - # Starts `count` sidekiq processes in the background. - def start_sidekiq_processes(count: 1) - Dir.chdir project_dir do - count.times { run_command start_sidekiq_cmd } - end - end - - # Stops sidekiq processes for the given pids, i.e. sends TERM signal. - def stop_processes(pids: []) - pids = active_pids & pids - pids.each { |pid| run_command stop_sidekiq_cmd(pid) } - end - - # Quiets sidekiq processes for the given pids, i.e. sends -TSTP signal. - def quiet_processes(pids: []) - pids = active_pids & pids - pids.each { |pid| run_command quiet_sidekiq_cmd(pid) } - end - - # Runs a block repeatedly for the given number of seconds, sleeping in - # between calls for the given interval, and returns when the given number - # of seconds has transpired, or when the block returns a truthy value, - # whichever comes first. - # @param seconds [Integer] The number of seconds to try the block until. - # @param interval [Integer] The number of additonal seconds to sleep - # before trying the block again. - # @param wait_msg [String] optional text to output when waiting to run - # the block again. - # @return value of the block - def try_until(seconds, interval: 5, wait_msg: "Waiting...") - block_val = yield - return if block_val - stop_time = Time.now.to_i + [seconds.to_i, 0].max - while !block_val && stop_time > Time.now.to_i - logger.info wait_msg if wait_msg - sleep interval if interval.to_i > 0 - block_val = yield - end - block_val - end - - def worker_pids - Sidekiq::Workers.new.map { |pid, thread_id, work| pid.split(':')[1] } - end - - # Log the command, then run it with backticks. - def run_command(command) - logger.info "Running: #{command}" - `#{command}` - end - - def start_sidekiq_cmd - "bundle exec sidekiq -e #{env} >> #{sidekiq_log_file} 2>&1 &" - end - - def sidekiq_log_file - "log/sidekiq.log" - end - - def stop_sidekiq_cmd(pid) - "kill #{pid}" - end - - def quiet_sidekiq_cmd(pid) - "kill -TSTP #{pid}" - end - - def default_config_file - File.expand_path('../../../config/sidekiq.yml', __FILE__) - end - - def project_dir - File.expand_path('../../../', __FILE__) - end - - def env - ENV['RAILS_ENV'] || 'development' - end - end -end From 73022df27240423218882b24f6d7f5df0e4d213e Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Fri, 8 Sep 2023 17:05:26 -0400 Subject: [PATCH 7/8] Removes DEPENDENCIES_NEXT env var from github builds --- .github/workflows/build-test-lint.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-test-lint.yaml b/.github/workflows/build-test-lint.yaml index 96a189d9..5820118a 100644 --- a/.github/workflows/build-test-lint.yaml +++ b/.github/workflows/build-test-lint.yaml @@ -201,7 +201,6 @@ jobs: env: HEAD_TAG: ${{ github.event.pull_request.head.sha || github.sha }} SE_NODE_SESSION_TIMEOUT: 999999 - DEPENDENCIES_NEXT: 1 BUNDLE_GEMFILE: Gemfile_next BUNDLER_VERSION: 2.0.1 shell: bash @@ -312,7 +311,6 @@ jobs: BUNDLER_VERSION: 2.0.1 CONFDIR: "/app/samvera/hyrax-webapp/solr/config" DB_CLEANER_ALLOW_REMOTE_DB_URL: "true" - DEPENDENCIES_NEXT: 1 SE_NODE_SESSION_TIMEOUT: 999999 TB_RSPEC_FORMATTER: progress TB_RSPEC_OPTIONS: --format RspecJunitFormatter --out rspec.xml @@ -353,7 +351,7 @@ jobs: - name: Setup db run: >- docker-compose exec -T web bash -c - "DEPENDENCIES_NEXT=1 RAILS_ENV=test bundle exec rake db:schema:load db:migrate db:seed" + "RAILS_ENV=test bundle exec rake db:schema:load db:migrate db:seed" - name: Run Specs id: run-specs env: @@ -365,8 +363,8 @@ jobs: continue-on-error: true run: >- docker-compose exec -T web bash -c - "DEPENDENCIES_NEXT=1 gem install semaphore_test_boosters && - DEPENDENCIES_NEXT=1 rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" + "gem install semaphore_test_boosters && + rspec_booster --job $CI_NODE_INDEX/$CI_NODE_TOTAL" - name: Fail job if spec failure run: if [[ ${{ steps.run-specs.outcome }} == "failure" ]]; then exit 1; else exit 0; fi - name: Publish Test Report From 885ba24617bc2ab135d7ed3091c9e1d6c50ce179 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Mon, 11 Dec 2023 17:05:47 -0800 Subject: [PATCH 8/8] use the hyrax helpers for ar style look ups and fix contributors not getting saved in bulkrax --- Gemfile.lock | 2 +- app/matchers/concerns/has_ams_matchers.rb | 1 + app/models/asset_resource.rb | 1 + app/models/contribution_resource.rb | 1 + app/models/digital_instantiation_resource.rb | 1 + app/models/essence_track_resource.rb | 1 + app/models/physical_instantiation_resource.rb | 1 + 7 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d46bacaa..02cc43de 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -41,7 +41,7 @@ GIT GIT remote: https://github.com/samvera/hyrax.git - revision: 43165fe0e360a60bf5470db1d3d55538217fcc7b + revision: a904e5a07040c148e4d6c3a9288d048efc8e42b9 branch: double_combo specs: hyrax (5.0.0.rc1) diff --git a/app/matchers/concerns/has_ams_matchers.rb b/app/matchers/concerns/has_ams_matchers.rb index 36c4043c..a00ff6d0 100644 --- a/app/matchers/concerns/has_ams_matchers.rb +++ b/app/matchers/concerns/has_ams_matchers.rb @@ -13,6 +13,7 @@ def field_supported?(field) return true if field == "title" return true if field == "description" return true if field == "subject" + return true if field == "contributors" property_defined = factory_class.singleton_methods.include?(:properties) && factory_class.properties[field].present? diff --git a/app/models/asset_resource.rb b/app/models/asset_resource.rb index 759b5735..535fd362 100644 --- a/app/models/asset_resource.rb +++ b/app/models/asset_resource.rb @@ -5,6 +5,7 @@ class AssetResource < Hyrax::Work include Hyrax::Schema(:basic_metadata) include Hyrax::Schema(:asset_resource) + include Hyrax::ArResource include AMS::WorkBehavior include AMS::CreateMemberMethods diff --git a/app/models/contribution_resource.rb b/app/models/contribution_resource.rb index adf0b34c..7e95d2d2 100644 --- a/app/models/contribution_resource.rb +++ b/app/models/contribution_resource.rb @@ -5,6 +5,7 @@ class ContributionResource < Hyrax::Work include Hyrax::Schema(:basic_metadata) include Hyrax::Schema(:contribution_resource) + include Hyrax::ArResource include AMS::WorkBehavior self.valid_child_concerns = [] diff --git a/app/models/digital_instantiation_resource.rb b/app/models/digital_instantiation_resource.rb index ac2d4fa9..a536395f 100644 --- a/app/models/digital_instantiation_resource.rb +++ b/app/models/digital_instantiation_resource.rb @@ -9,6 +9,7 @@ class DigitalInstantiationResource < Hyrax::Work include Hyrax::Schema(:basic_metadata) include Hyrax::Schema(:digital_instantiation_resource) + include Hyrax::ArResource include AMS::WorkBehavior include ::AMS::CreateMemberMethods extend CarrierWave::Mount diff --git a/app/models/essence_track_resource.rb b/app/models/essence_track_resource.rb index b3e3b545..dad0eaad 100644 --- a/app/models/essence_track_resource.rb +++ b/app/models/essence_track_resource.rb @@ -5,6 +5,7 @@ class EssenceTrackResource < Hyrax::Work include Hyrax::Schema(:basic_metadata) include Hyrax::Schema(:essence_track_resource) + include Hyrax::ArResource include AMS::WorkBehavior self.valid_child_concerns = [] diff --git a/app/models/physical_instantiation_resource.rb b/app/models/physical_instantiation_resource.rb index aa48a14e..225902f5 100644 --- a/app/models/physical_instantiation_resource.rb +++ b/app/models/physical_instantiation_resource.rb @@ -5,6 +5,7 @@ class PhysicalInstantiationResource < Hyrax::Work include Hyrax::Schema(:basic_metadata) include Hyrax::Schema(:physical_instantiation_resource) + include Hyrax::ArResource include AMS::WorkBehavior include ::AMS::CreateMemberMethods