diff --git a/sphinx/aws/aws_specify_asset_server_setup.md b/sphinx/aws/aws_specify_asset_server_setup.md index 61e3943..59e038e 100644 --- a/sphinx/aws/aws_specify_asset_server_setup.md +++ b/sphinx/aws/aws_specify_asset_server_setup.md @@ -1,140 +1,19 @@ # AWS Specify Asset Server Setup ## EC2 Non-Dockerized Build -```bash -#!/bin/bash - -sudo apt update; -sudo apt upgrade -y; -sudo apt-get -y install --no-install-recommends \ - python3-venv \ - python3.8 \ - python3.8-dev \ - python3-pip \ - imagemagick \ - ghostscript \ - git \ - nginx \ - certbot \ - authbind \ - s3fs \ - awscli; - -# python 3.6 -#sudo apt update -#sudo apt install build-essential checkinstall zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev libsqlite3-dev wget libbz2-dev; -#wget https://www.python.org/ftp/python/3.6.15/Python-3.6.15.tgz; -#tar -xf Python-3.6.15.tgz; -#cd Python-3.6.15 && ./configure --enable-optimizations; -#make -j$(nproc); -#sudo make altinstall; -#python3.6 --version; - -# python 3.6 install with apt -sudo apt install software-properties-common; -sudo add-apt-repository ppa:deadsnakes/ppa; -sudo apt update; -sudo apt install python3.6; -sudo apt-get install python3.6-distutils; - -# install pip3.6 -#wget https://bootstrap.pypa.io/pip/3.6/get-pip.py; -python3.6 -m venv --without-pip ve; -source ve/bin/activate; -wget https://bootstrap.pypa.io/get-pip.py; -#wget https://bootstrap.pypa.io/pip/3.5/get-pip.py -#deactivate; - -# activate python3.6 venv -sudo apt install -y python3-virtualenv; -python3.6 -m venv myenv; -source myenv/bin/activate; -pip install --no-cache-dir -r requirements.txt; -#deactivate; - -# TLS dependencies -sudo apt-get -y install --no-install-recommends \ - certbot \ - python3-certbot-nginx \ - software-properties-common; - -# Configure AWS -aws configure set aws_access_key_id "ACCESS_KEY"; -aws configure set aws_secret_access_key "ACCESS_KEY_SECRET"; -aws configure set default.region us-east-1; -aws configure set default.output json; - -# Import attachment files -#mkdir attachments; -#aws s3 cp s3://specify-cloud/assets-server/attachments/ ~/attachments --recursive; -# S3 Mounting -mkdir attachments; -s3fs specify-cloud /assets-server/attachments/; +Make sure to set environment variables before running the bash script -# Clone asset server repo -git clone https://github.com/specify/web-asset-server.git; -cd ~/web-asset-server; -git checkout arm-build; - -# Build python web asset server -python3.8 -m venv ve; -sudo ve/bin/pip install --no-cache-dir -r requirements.txt -#sudo pip install -r requirements.txt; - -# Port config -# not needed when running with nginx -#sudo apt-get install authbind; -#touch 80; -#chmod u+x 80; -#sudo mv 80 /etc/authbind/byport; - -# Create SystemD service -sudo cat > /etc/systemd/system/web-asset-server.service << EOF -[Unit] -Description=Specify Web Asset Server -Wants=network.target - -[Service] -User=ubuntu -WorkingDirectory=/home/ubuntu/web-asset-server -ExecStart=/home/ubuntu/web-asset-server/ve/bin/python /home/ubuntu/web-asset-server/server.py -Restart=always - -[Install] -WantedBy=multi-user.target - -EOF - -sudo systemctl daemon-reload; -sudo systemctl enable web-asset-server.service; -sudo systemctl start web-asset-server.service; -sudo systemctl status web-asset-server.service; - -# nginx -# sudo vim etc/nginx/sites-enabled/assets.conf -sudo rm -f /etc/nginx/sites-enabled/default; -sudo nginx -t; -sudo /etc/init.d/nginx reload; - -# S3 Mounting -mount -o discard,defaults,noatime /dev/disk/by-id/scsi-0DO_Volume_volume-nyc1-01 /mnt/volume-nyc1-01 - -# TODO: EFS Mounting - -# Certbot TLS config -sudo mkdir /var/www/.well-known; -sudo certbot --nginx -d assets-test.specifycloud.org -d assets-test.specifycloud.org; -sudo ls -la /etc/letsencrypt/live/assets-test.specifycloud.org; -#openssl dhparam -out /etc/nginx/dhparam.pem 4096; -sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096; #2048 or 1024 -sudo openssl dhparam -dsaparam -out /etc/ssl/certs/dhparam.pem 1024; -# add https server config to nginx assets. - -# Edit +```command line +export DOMAIN_NAME +export SUBDOMAIN_PREFIX ``` +[build_non_docker_asset_server.sh](../scripts/build_non_docker_asset_server.sh) to build the instance. + ## Config files + +Make sure to set environment variables /etc/systemd/system/web-asset-server.service -> ``` [Unit] @@ -185,7 +64,7 @@ ALLOW_STATIC_FILE_ACCESS = True # These values are interpolated into the web_asset_store.xml resource # so the client knows how to talk to the server. #HOST = 'localhost' -HOST = 'assets-test.specifycloud.org' +HOST = 'subdomain.domain.name' PORT = 8080 #PORT = 80 @@ -228,7 +107,7 @@ CAN_THUMBNAIL = {'image/jpeg', 'image/gif', 'image/png', 'image/tiff', 'applicat SERVER = 'wsgiref' # For testing. Requires no extra packages. ``` -/etc/nginx/sites-enabled/assets.conf from the aasets1.specifycloud.org- -> +/etc/nginx/sites-enabled/assets.conf from the subdomain.domain.name- -> ``` # Nginx configuration for supplying an HTTPS end point for the web # asset server. The asset server is running on the same system @@ -241,7 +120,7 @@ SERVER = 'wsgiref' # For testing. Requires no extra packages. server { # HTTP access is needed for Specify 6. It will not work with HTTPS. listen 80 default_server; - server_name assets1.specifycloud.org; + server_name subdomain.domain.name; client_max_body_size 0; # The LetsEncrypt certificate mechanism places a nonce @@ -258,7 +137,7 @@ server { # it defines point to this proxy. location = /web_asset_store.xml { proxy_pass http://localhost:8080/web_asset_store.xml; - sub_filter 'http://assets1.specifycloud.org:8080' 'http://assets1.specifycloud.org'; + sub_filter 'http://subdomain.domain.name:8080' 'http://subdomain.domain.name'; sub_filter_once off; sub_filter_types text/xml; } @@ -273,11 +152,11 @@ server { server { # This stanza defines the HTTPS end point. listen 443 ssl default_server; - server_name assets1.specifycloud.org; + server_name subdomain.domain.name; client_max_body_size 0; - ssl_certificate /etc/letsencrypt/live/assets1.specifycloud.org/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/assets1.specifycloud.org/privkey.pem; + ssl_certificate /etc/letsencrypt/live/subdomain.domain.name/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/subdomain.domain.name/privkey.pem; # from https://cipherli.st/ # and https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html @@ -311,7 +190,7 @@ server { # to use HTTPS in addition to changing the port. location = /web_asset_store.xml { proxy_pass http://localhost:8080/web_asset_store.xml; - sub_filter 'http://assets1.specifycloud.org:8080' 'https://assets1.specifycloud.org'; + sub_filter 'http://subdomain.domain.name:8080' 'https://subdomain.domain.name'; sub_filter_once off; sub_filter_types text/xml; } @@ -323,15 +202,15 @@ server { } ``` -/etc/letsencrypt/renewal/assets1.specifycloud.org.conf -> +/etc/letsencrypt/renewal/subdomain.domain.name.conf -> ``` # renew_before_expiry = 30 days -cert = /etc/letsencrypt/live/assets1.specifycloud.org/cert.pem -privkey = /etc/letsencrypt/live/assets1.specifycloud.org/privkey.pem -chain = /etc/letsencrypt/live/assets1.specifycloud.org/chain.pem -fullchain = /etc/letsencrypt/live/assets1.specifycloud.org/fullchain.pem +cert = /etc/letsencrypt/live/subdomain.domain.name/cert.pem +privkey = /etc/letsencrypt/live/subdomain.domain.name/privkey.pem +chain = /etc/letsencrypt/live/subdomain.domain.name/chain.pem +fullchain = /etc/letsencrypt/live/subdomain.domain.name/fullchain.pem version = 1.9.0 -archive_dir = /etc/letsencrypt/archive/assets1.specifycloud.org +archive_dir = /etc/letsencrypt/archive/subdomain.domain.name # Options and defaults used in the renewal process [renewalparams] @@ -340,18 +219,13 @@ account = a563615cc912ed3d7a3edfede09d6760 post_hook = systemctl reload nginx server = https://acme-v02.api.letsencrypt.org/directory [[webroot_map]] -assets1.specifycloud.org = /var/www +subdomain.domain.name = /var/www ``` -/etc/ssl/certs/dhparam.pem from assets1.specofycloud.org-> +/etc/ssl/certs/dhparam.pem from subdomain.domain.name-> ``` -----BEGIN DH PARAMETERS----- -MIIBCAKCAQEAlcFKsIuFylwX47jxqbNT0wSVD6ifznsMcti8f7T+zaQQNr84IYIM -pNTT9E6SrVkkJg2u1nGScNqj5lArXvrda6zL66T8WmkFFrGfNW7RYCQ3vpg6BpGs -dJ3+HtWYDNoMbeCrDyMz1DDfX/3OWblTTZRbjpvn/tEgTAn3DexP/QkE9E2c1AUX -Mf/07vWpZ7giemaNgaME3fHDKyReNhTpfg1eDKypUUhEmr+PJmWQ9LQBc12LyXOP -DaFwAJUrqwEqrQP5fEQdOMdh522RwuD2/fPeXTukQHI8gUuMjk652aeLOcn1Ufhy -/KbbV6TJi7wS5F3HVaNXGOLMsHq+CywOCwIBAg== +... -----END DH PARAMETERS----- ``` diff --git a/sphinx/aws/aws_specify_cloud_setup.md b/sphinx/aws/aws_specify_cloud_setup.md index 444e39b..4a9eddc 100644 --- a/sphinx/aws/aws_specify_cloud_setup.md +++ b/sphinx/aws/aws_specify_cloud_setup.md @@ -9,66 +9,10 @@ EC2 Parameters: ami-0770bf1d6ae61c858 ## Initial Commands -```bash -#!/bin/bash -sudo apt-get update; -sudo apt upgrade; -sudo apt install -y make python3-pip - -#sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose mysql-client; -#sudo apt install docker.io - -# Install Docker -sudo apt install -y apt-transport-https ca-certificates curl software-properties-common; -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg; -echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null; -sudo apt update; -sudo apt -y install docker-ce; -docker --version; -sudo systemctl status docker; - -#sudo systemctl start docker.service -#sudo systemctl enable docker.service -#sudo systemctl status docker.service - -# Install docker compose -mkdir .docker -mkdir .docker/cli-plugins -curl -L "https://github.com/docker/compose/releases/download/v2.17.2/docker-compose-linux-$(uname -m)" -o ~/.docker/cli-plugins/docker-compose; -chmod +x ~/.docker/cli-plugins/docker-compose; -docker compose version; - -# Install awscli -sudo apt -y install awscli; -aws configure; # See aws credentials at bottom of notes - -# Setup mysql datbase connection -sudo apt install -y mysql-client; - -# Copy files from S3 -aws s3 cp s3://specify-cloud/repo-snapshots/docker-compositions/ ./ --recursive - -# python setup -sudo apt install -y python3-pip; -python3 -m pip install j2cli; - -# Create config files for specifycloud -cd specifycloud; -vim spcloudservers.json -vim defaults.env -sudo apt install j2cli -make - -# ssh client-server alive settings -sudo echo -e "ClientAliveInterval 1200\nClientAliveCountMax 3" >> /etc/ssh/sshd_config -sudo systemctl reload sshd; - -#docker pull specifyconsortium/specify7-service:edge - -# su specify -c make -docker-compose up -d -``` +Initial commands are in the script +[specify_cloud_setup.sh](../../scripts/specify_cloud_setup.sh) + ## SSH Configuration @@ -100,8 +44,8 @@ spcloudservers.json -> "sp6": "specify6803", "https": false, "env": { - "ASSET_SERVER_URL": "https://demo-assets.specifycloud.org/web_asset_store.xml", - "ANONYMOUS_USER": "sp7demofish" + "ASSET_SERVER_URL": "https://./web_asset_store.xml", + "ANONYMOUS_USER": "anon_user_name" } } }, @@ -116,13 +60,13 @@ spcloudservers.json -> ``` defaults.env -> ``` -DATABASE_HOST=specifycloud-dev-database-1-instance-1.cqvncffkwz9t.us-east-1.rds.amazonaws.com +DATABASE_HOST=...rds.amazonaws.com DATABASE_PORT=3306 MASTER_NAME=master -MASTER_PASSWORD=mastermaster +MASTER_PASSWORD= SECRET_KEY=bogus -ASSET_SERVER_URL=https://assets1.specifycloud.org/web_asset_store.xml -ASSET_SERVER_KEY=tnhercbrhtktanehul.dukb +ASSET_SERVER_URL=https://ASSET_SERVER_FQDN/web_asset_store.xml +ASSET_SERVER_KEY=ASSET_SERVER_KEY REPORT_RUNNER_HOST=10.132.218.32 REPORT_RUNNER_PORT=8080 CELERY_BROKER_URL=redis://redis/0 @@ -135,95 +79,25 @@ SP7_DEBUG=false ## Info Misc. ### aws credentials: -- username: `specify.user` +- username: SPECIFY_USER - password: SPECIFY_USER_PASSWORD - access key: ACCESS_KEY - secret access key: ACCESS_KEY_SECRET -- default region: us-east-1 +- default region: REGION - default output format: json ### AWS EC2 User data: -```bash -# Avoid services restarting during apt upgrade -sudo sed -i "s/#\$nrconf{kernelhints} = -1;/\$nrconf{kernelhints} = -1;/g" /etc/needrestart/needrestart.conf; -sudo sed -i "s/#\$nrconf{restart} = 'i';/\$nrconf{restart} = 'a';/g" /etc/needrestart/needrestart.conf; - -# Run apt installs -sudo apt update; -sudo apt upgrade -y; -sudo apt install -y apt-transport-https ca-certificates git gh curl software-properties-common wget python3-pip awscli mysql-client j2cli; - -# Configure AWS -aws configure set aws_access_key_id "ACCESS_KEY"; -aws configure set aws_secret_access_key "ACCESS_KEY_SECRET"; -aws configure set default.region us-east-1; -aws configure set default.output json; - -# Install Docker -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg; -echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null; -sudo apt update; -apt-cache policy docker-ce; -sudo apt install -y docker-ce; -docker --version; - -# Install docker compose -mkdir .docker; -mkdir .docker/cli-plugins; -curl -L "https://github.com/docker/compose/releases/download/v2.17.2/docker-compose-linux-$(uname -m)" -o ~/.docker/cli-plugins/docker-compose; -chmod +x ~/.docker/cli-plugins/docker-compose; -docker compose version; - -# Python setup -# python3 -m pip install j2cli; -# export PATH=$PATH:/home/ubuntu/.local/bin; -# sudo apt install j2cli; - -# Copy files from S3 -aws s3 cp s3://specify-cloud/repo-snapshots/docker-compositions/ ./ --recursive; -aws s3 cp s3://specify-cloud/repo-snapshots/spcloudservers.json ./specifycloud/; -aws s3 cp s3://specify-cloud/repo-snapshots/defaults.env ./specifycloud/; - -# Database setup -mkdir seed-databases; -aws s3 cp s3://specify-cloud/seed-database/specify.sql ./seed-databases/; -mysql --host specify-cloud-swiss-demo-database-1.c9qlnkmf2lfl.eu-central-2.rds.amazonaws.com --port 3306 -u master -p'mastermaster' -e "create database specify;"; -mysql --host specify-cloud-swiss-demo-database-1.c9qlnkmf2lfl.eu-central-2.rds.amazonaws.com --port 3306 -u master -p'mastermaster' specify < ./seed-databases/specify.sql; -rm -f ./seed-databases/specify.sql; - -# Configure Specify Network -cd specifycloud; -touch spcloudservers.json; -touch defaults.env; - -# Run Specify Network -make; -sudo docker compose up -d; - -# Certbot setup -sudo apt install certbot python3-certbot-apache; -sudo mkdir /var/www/sp7demofish; - -# Github clone private repo -ssh-keygen -t ed25519 -C "acwhite211@gmail.com"; -#ssh-keygen -t rsa -b 4096 -C "acwhite211@gmail.com"; - -# git clone repos -git clone https://github.com/specify/specify7.git; -git clone https://github.com/specify/specify6.git; -git clone https://github.com/specify/report-runner-service.git; -#git clone https://github.com/specify/web-asset-server.git; - -# Install nginx -sudo apt install -y nginx openjdk-8-jdk maven ant; -sudo ufw allow 'Nginx HTTP'; -sudo ufw status; -sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-arm64/jre/bin/java; - -# Build without docker -cd specify6; -ant compile-nonmac; -``` + +Make sure to fill in all variables (starting with `$`) in the following userdata script +script before including it in an EC2 launch configuration. +[specify_cloud_ec2_userdata.sh](../../scripts/specify_cloud_ec2_userdata.sh) + +* $BUCKET_NAME +* $DATABASE_NAME +* $DB_IDENTIFIER +* $REGION +* $MASTER_PASSWORD + ### AWS Pricing @@ -311,130 +185,17 @@ t4g.medium - 36 month reserved instance -### Specify Network Extract - -Specify Network EC2 instance: -```bash -sudo apt update; -sudo apt upgrade -y; -sudo apt install -y wget awscli unzip; -aws configure set aws_access_key_id "ACCESS_KEY"; -aws configure set aws_secret_access_key "ACCESS_KEY_SECRET"; -aws configure set default.region us-east-1; -aws configure set default.output json; -mkdir gbif; -mkdir gbif/download; -mkdir gbif/extract; -cd gbif/download; -aws s3 cp s3://specify-network/gbif/0146304-230224095556074.zip ./; -wget $GBIF_URL; -unzip 0146304-230224095556074.zip -d ../extract/; -cd ../extract/; -mv ./0146304-230224095556074.csv ./gbif.csv -aws s3 cp gbif.csv s3://specify-network-dev/gbif_test/gbif_extract/; -``` - -specify aws github ssh key: -id_ed25519.pub -> -``` -sh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKHq3lVhZ4U8j0Derpm37wgUPLGLgQtim77M68m+XNWL acwhite211@gmail.com -``` -id_ed25519 -> -``` ------BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW -QyNTUxOQAAACCh6t5VYWeFPI9A3q6Zt+8IFDyxi4ELYpu+zOvJvlzViwAAAJj4E1iO+BNY -jgAAAAtzc2gtZWQyNTUxOQAAACCh6t5VYWeFPI9A3q6Zt+8IFDyxi4ELYpu+zOvJvlzViw -AAAEDi0KTenAzeyomMyaqOBd8APyQjcL3YU7tXMMMrit8bjaHq3lVhZ4U8j0Derpm37wgU -PLGLgQtim77M68m+XNWLAAAAFGFjd2hpdGUyMTFAZ21haWwuY29tAQ== ------END OPENSSH PRIVATE KEY----- -``` - -MariaDB version: 10.3.38-MariaDB-0ubuntu0.20.04.1-log -AWS DB password: DB_PASSWORD - ### Install Ubuntu EC2 instance with no docker -```bash -#!/bin/bash -# Avoid services restarting during apt upgrade -sudo sed -i "s/#\$nrconf{kernelhints} = -1;/\$nrconf{kernelhints} = -1;/g" /etc/needrestart/needrestart.conf; -sudo sed -i "s/#\$nrconf{restart} = 'i';/\$nrconf{restart} = 'a';/g" /etc/needrestart/needrestart.conf; +Make sure to fill in all variables (starting with `$`) in the following userdata script +script before including it in an EC2 launch configuration. +[install_ec2_wo_docker.sh](../../scripts/install_ec2_wo_docker.sh) -# Run apt installs -sudo apt update; -sudo apt upgrade -y; -sudo apt install -y --no-install-recommends \ - apt-transport-https ca-certificates git curl software-properties-common wget \ - python3-pip awscli mysql-client j2cli nginx openjdk-8-jdk maven ant gcc make \ - openldap-devel \ - #nodejs npm \ - python3-venv \ - #python3.8 python3.8-dev \ - redis unzip \ - apache2 \ - libapache2-mod-wsgi-py3; - -# Install nodejs 18 -cd ~; -#curl -sL https://deb.nodesource.com/setup_18.x -o nodesource_setup.sh; -#sudo bash nodesource_setup.sh; -#sudo apt install -y nodejs; -curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash; -source ~/.bashrc; -nvm install v18; -nvm use 18; -nvm alias default 18; -node -v; - -# Install python 3.8 -sudo add-apt-repository -y ppa:deadsnakes/ppa; -sudo apt update; -sudo apt install -y python3.8 python3.8-dev; - -# Git clone repos -git clone https://github.com/specify/specify7.git; -git clone https://github.com/specify/specify6.git; -git clone https://github.com/specify/report-runner-service.git; - -# Setup database -aws s3 cp s3://specify-cloud/seed-database/sp7demofish.sql ./specify7/seed-database/; - -# Setup specify6 -#cd ~/specify6; -#sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-arm64/jre/bin/java; -#wget https://update.specifysoftware.org/6803/Specify_unix_64.sh; -#sh Specify_unix_64.sh -q -dir ./Specify6.8.03; -#sudo ln -s $(pwd)/Specify6.8.03 /opt/Specify; -sudo ln -s $(pwd)/specify6 /opt/Specify; - -# Setup specify7 -cd ~/specify7; -#git checkout tags/v7.8.10 -python3.8 -m venv specify7/ve; -specify7/ve/bin/pip install wheel; -specify7/ve/bin/pip install --upgrade -r specify7/requirements.txt; - -# Run specify dev -cd ~/specify7; -source ve/bin/activate; -make runserver; - -# Setup specify-worker -cd ~/specify7; -#ve/bin/celery -A specifyweb worker -l INFO --concurrency=1 -Q specify; -celery -A specifyweb worker -l INFO --concurrency=1; +* $BUCKET_NAME -# Setup apache -# sudo apt install -y apache2 libapache2-mod-wsgi-py3 - -# Setup nginx -#sudo apt install -y nginx openjdk-8-jdk maven ant; -sudo ufw allow 'Nginx HTTP'; -sudo ufw status; -``` ### Using the Amazon arm54 centos image: + ```bash #!/bin/bash diff --git a/sphinx/aws/client_migration_notes.md b/sphinx/aws/client_migration_notes.md index e18cef8..be6cdba 100644 --- a/sphinx/aws/client_migration_notes.md +++ b/sphinx/aws/client_migration_notes.md @@ -22,7 +22,7 @@ On Windows PuTTY target ```commandline C:\Program Files\PuTTY\putty.exe -ssh -i C:\users\your_user\private_key_.ppk \ - institution_id@eu-db-1.specifycloud.org -L 3307:xxx.xx.xx.xx:3306 -N` + institution_id@.specifycloud.org -L 3307:xxx.xx.xx.xx:3306 -N` ``` For now, you will log into the database as `master` with the same previous passwords, diff --git a/sphinx/aws/ec2_rds_specify7_setup.md b/sphinx/aws/ec2_rds_specify7_setup.md index 7918659..c100b3b 100644 --- a/sphinx/aws/ec2_rds_specify7_setup.md +++ b/sphinx/aws/ec2_rds_specify7_setup.md @@ -2,7 +2,7 @@ These are the detailed instructions to get Specify7 up and running on an EC2 instance. This is a temporary solution for deployment, with an ECS solution coming in the future. -These instruction require access to the `specify/docker-compositions` private repo. +These instruction require access to the `docker-compositions` private repo. For this kind of AWS deployment, it is easiest to use the `specifycloud` deployment, even with just one client instance. I used the `specifycloud` deployment for my AWS EC2 instance, but if you want to include things like the asset server, you can create a @@ -64,6 +64,10 @@ true only after you have setup SSL/TLS. The database value is the name of your database. ## Config file + +Edit the DATABASE_HOST, MASTER_PASSWORD, ASSET_SERVER_URL, ASSET_SERVER_KEY, +and RRHOST_PRIVATE_IP + defaults.env -> ``` DATABASE_HOST= @@ -73,15 +77,13 @@ MASTER_PASSWORD= SECRET_KEY=temp ASSET_SERVER_URL=https://assets1.specifycloud.org/web_asset_store.xml ASSET_SERVER_KEY= -REPORT_RUNNER_HOST=10.132.218.32 +REPORT_RUNNER_HOST= REPORT_RUNNER_PORT=8080 CELERY_BROKER_URL=redis://redis/0 CELERY_RESULT_BACKEND=redis://redis/1 LOG_LEVEL=WARNING SP7_DEBUG=false ``` -You will only need to edit the DATABASE_HOST, MASTER_PASSWORD, ASSET_SERVER_URL, and -ASSET_SERVER_KEY ## Configure Specify7 @@ -97,6 +99,15 @@ you can use the AWS Secrets manager if you prefer. Also note that after some of commands, SystemD services will restart. When that happens, just press enter once or twice to you get back to the shell. +Make sure to fill in all variables (starting with `$`) and YOUR_AWS_ACCESS_KEY* in the +following script before running commands + +* $SPECIFY_CLOUD_BUCKET_NAME +* $REGION +* YOUR_AWS_ACCESS_KEY_ID +* YOUR_AWS_ACCESS_KEY_SECRET + + ```bash # Avoid services restarting during apt updates sudo sed -i "s/#\$nrconf{kernelhints} = -1;/\$nrconf{kernelhints} = -1;/g" /etc/needrestart/needrestart.conf; @@ -109,7 +120,7 @@ sudo apt install -y apt-transport-https ca-certificates git gh curl software-pro # Configure AWS aws configure set aws_access_key_id "YOUR_AWS_ACCESS_KEY_ID"; aws configure set aws_secret_access_key "YOUR_AWS_ACCESS_KEY_SECRET"; -aws configure set default.region us-east-1; +aws configure set default.region $REGION; aws configure set default.output json; # Install Docker @@ -128,10 +139,10 @@ chmod +x ~/.docker/cli-plugins/docker-compose; docker compose version; # Check to make sure it installed correctly # Copy files from S3, only need to copy the file that you need -aws s3 cp s3://specify-cloud/repo-snapshots/docker-compositions/ ./ --recursive; -aws s3 cp s3://specify-cloud/config/github-access-key.txt ./; -aws s3 cp s3://specify-cloud/config/spcloudservers.json ./specifycloud/; -aws s3 cp s3://specify-cloud/config/defaults.env ./specifycloud/; +aws s3 cp s3://$SPECIFY_CLOUD_BUCKET_NAME/repo-snapshots/docker-compositions/ ./ --recursive; +aws s3 cp s3://$SPECIFY_CLOUD_BUCKET_NAME/config/github-access-key.txt ./; +aws s3 cp s3://$SPECIFY_CLOUD_BUCKET_NAME/config/spcloudservers.json ./specifycloud/; +aws s3 cp s3://$SPECIFY_CLOUD_BUCKET_NAME/config/defaults.env ./specifycloud/; # Optional, need to clone if you did not copy repo from S3 # Clone private repo, need to use github cli @@ -162,14 +173,22 @@ Get the database host from the AWS RDS webpage. In the connectivity tab of your instance, copy the test marked 'Endpoint'. Then run the following commands to upload your data to the database. +Make sure to fill in all variables (starting with `$`) in the +following script before running commands + +* $SPECIFY_CLOUD_BUCKET_NAME +* $SPECIFY_CLOUD_DATABASE_NAME +* $DB_IDENTIFIER +* $REGION + ```bash # Database setup cd specifycloud; mkdir seed-databases; -aws s3 cp s3://specify-cloud/seed-database/archireef.sql ./seed-databases/; -mysql --host specify-cloud-database-1...rds.amazonaws.com \ +aws s3 cp s3://$SPECIFY_CLOUD_BUCKET_NAME/seed-database/archireef.sql ./seed-databases/; +mysql --host $SPECIFY_CLOUD_DATABASE_NAME.$DB_IDENTIFIER.$REGION.rds.amazonaws.com \ --port 3306 -u master -p -e "create database archireef;"; -mysql --host specify-cloud-database-1...rds.amazonaws.com \ +mysql --host $SPECIFY_CLOUD_DATABASE_NAME.$DB_IDENTIFIER.$REGION.rds.amazonaws.com \ --port 3306 -u master -p specify < ./seed-databases/archireef.sql; rm -f ./seed-databases/archireef.sql; ``` diff --git a/sphinx/scripts/build_non_docker_asset_server.sh b/sphinx/scripts/build_non_docker_asset_server.sh new file mode 100644 index 0000000..f1a56f9 --- /dev/null +++ b/sphinx/scripts/build_non_docker_asset_server.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +## Set these environment variables before running the script +# export DOMAIN_NAME +# export SUBDOMAIN_PREFIX + +sudo apt update; +sudo apt upgrade -y; +sudo apt-get -y install --no-install-recommends \ + python3-venv \ + python3.8 \ + python3.8-dev \ + python3-pip \ + imagemagick \ + ghostscript \ + git \ + nginx \ + certbot \ + authbind \ + s3fs \ + awscli; + +# python 3.6 +#sudo apt update +#sudo apt install build-essential checkinstall zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev libsqlite3-dev wget libbz2-dev; +#wget https://www.python.org/ftp/python/3.6.15/Python-3.6.15.tgz; +#tar -xf Python-3.6.15.tgz; +#cd Python-3.6.15 && ./configure --enable-optimizations; +#make -j$(nproc); +#sudo make altinstall; +#python3.6 --version; + +# python 3.6 install with apt +sudo apt install software-properties-common; +sudo add-apt-repository ppa:deadsnakes/ppa; +sudo apt update; +sudo apt install python3.6; +sudo apt-get install python3.6-distutils; + +# install pip3.6 +#wget https://bootstrap.pypa.io/pip/3.6/get-pip.py; +python3.6 -m venv --without-pip ve; +source ve/bin/activate; +wget https://bootstrap.pypa.io/get-pip.py; +#wget https://bootstrap.pypa.io/pip/3.5/get-pip.py +#deactivate; + +# activate python3.6 venv +sudo apt install -y python3-virtualenv; +python3.6 -m venv myenv; +source myenv/bin/activate; +pip install --no-cache-dir -r requirements.txt; +#deactivate; + +# TLS dependencies +sudo apt-get -y install --no-install-recommends \ + certbot \ + python3-certbot-nginx \ + software-properties-common; + +# Configure AWS +aws configure set aws_access_key_id "ACCESS_KEY"; +aws configure set aws_secret_access_key "ACCESS_KEY_SECRET"; +aws configure set default.region us-east-1; +aws configure set default.output json; + +# Import attachment files +#mkdir attachments; +#aws s3 cp s3://specify-cloud/assets-server/attachments/ ~/attachments --recursive; + +# S3 Mounting +mkdir attachments; +s3fs specify-cloud /assets-server/attachments/; + +# Clone asset server repo +git clone https://github.com/specify/web-asset-server.git; +cd ~/web-asset-server; +git checkout arm-build; + +# Build python web asset server +python3.8 -m venv ve; +sudo ve/bin/pip install --no-cache-dir -r requirements.txt +#sudo pip install -r requirements.txt; + +# Port config +# not needed when running with nginx +#sudo apt-get install authbind; +#touch 80; +#chmod u+x 80; +#sudo mv 80 /etc/authbind/byport; + +# Create SystemD service +sudo cat > /etc/systemd/system/web-asset-server.service << EOF +[Unit] +Description=Specify Web Asset Server +Wants=network.target + +[Service] +User=ubuntu +WorkingDirectory=/home/ubuntu/web-asset-server +ExecStart=/home/ubuntu/web-asset-server/ve/bin/python /home/ubuntu/web-asset-server/server.py +Restart=always + +[Install] +WantedBy=multi-user.target + +EOF + +sudo systemctl daemon-reload; +sudo systemctl enable web-asset-server.service; +sudo systemctl start web-asset-server.service; +sudo systemctl status web-asset-server.service; + +# nginx +# sudo vim etc/nginx/sites-enabled/assets.conf +sudo rm -f /etc/nginx/sites-enabled/default; +sudo nginx -t; +sudo /etc/init.d/nginx reload; + +# S3 Mounting +mount -o discard,defaults,noatime /dev/disk/by-id/scsi-0DO_Volume_volume-nyc1-01 /mnt/volume-nyc1-01 + +# TODO: EFS Mounting + +sudo ls -la /etc/letsencrypt/live/$SUBDOMAIN_PREFIX.$DOMAIN_NAME; +# Certbot TLS config +sudo mkdir /var/www/.well-known; +sudo certbot --nginx -d $SUBDOMAIN_PREFIX.$DOMAIN_NAME -d $SUBDOMAIN_PREFIX.$DOMAIN_NAME; +sudo ls -la /etc/letsencrypt/live/$SUBDOMAIN_PREFIX.$DOMAIN_NAME; +#openssl dhparam -out /etc/nginx/dhparam.pem 4096; +sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096; #2048 or 1024 +sudo openssl dhparam -dsaparam -out /etc/ssl/certs/dhparam.pem 1024; +# add https server config to nginx assets. + +# Edit \ No newline at end of file diff --git a/sphinx/scripts/install_ec2_wo_docker.sh b/sphinx/scripts/install_ec2_wo_docker.sh new file mode 100644 index 0000000..127d670 --- /dev/null +++ b/sphinx/scripts/install_ec2_wo_docker.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# Replace the following variable before running this script: +# $BUCKET_NAME + +# Avoid services restarting during apt upgrade +sudo sed -i "s/#\$nrconf{kernelhints} = -1;/\$nrconf{kernelhints} = -1;/g" /etc/needrestart/needrestart.conf; +sudo sed -i "s/#\$nrconf{restart} = 'i';/\$nrconf{restart} = 'a';/g" /etc/needrestart/needrestart.conf; + +# Run apt installs +sudo apt update; +sudo apt upgrade -y; +sudo apt install -y --no-install-recommends \ + apt-transport-https ca-certificates git curl software-properties-common wget \ + python3-pip awscli mysql-client j2cli nginx openjdk-8-jdk maven ant gcc make \ + openldap-devel \ + #nodejs npm \ + python3-venv \ + #python3.8 python3.8-dev \ + redis unzip \ + apache2 \ + libapache2-mod-wsgi-py3; + +# Install nodejs 18 +cd ~; +#curl -sL https://deb.nodesource.com/setup_18.x -o nodesource_setup.sh; +#sudo bash nodesource_setup.sh; +#sudo apt install -y nodejs; +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash; +source ~/.bashrc; +nvm install v18; +nvm use 18; +nvm alias default 18; +node -v; + +# Install python 3.8 +sudo add-apt-repository -y ppa:deadsnakes/ppa; +sudo apt update; +sudo apt install -y python3.8 python3.8-dev; + +# Git clone repos +git clone https://github.com/specify/specify7.git; +git clone https://github.com/specify/specify6.git; +git clone https://github.com/specify/report-runner-service.git; + +# Setup database +aws s3 cp s3://$BUCKET_NAME/seed-database/sp7demofish.sql ./specify7/seed-database/; + +# Setup specify6 +#cd ~/specify6; +#sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-arm64/jre/bin/java; +#wget https://update.specifysoftware.org/6803/Specify_unix_64.sh; +#sh Specify_unix_64.sh -q -dir ./Specify6.8.03; +#sudo ln -s $(pwd)/Specify6.8.03 /opt/Specify; +sudo ln -s $(pwd)/specify6 /opt/Specify; + +# Setup specify7 +cd ~/specify7; +#git checkout tags/v7.8.10 +python3.8 -m venv specify7/ve; +specify7/ve/bin/pip install wheel; +specify7/ve/bin/pip install --upgrade -r specify7/requirements.txt; + +# Run specify dev +cd ~/specify7; +source ve/bin/activate; +make runserver; + +# Setup specify-worker +cd ~/specify7; +#ve/bin/celery -A specifyweb worker -l INFO --concurrency=1 -Q specify; +celery -A specifyweb worker -l INFO --concurrency=1; + +# Setup apache +# sudo apt install -y apache2 libapache2-mod-wsgi-py3 + +# Setup nginx +#sudo apt install -y nginx openjdk-8-jdk maven ant; +sudo ufw allow 'Nginx HTTP'; +sudo ufw status; diff --git a/sphinx/scripts/specify_cloud_ec2_userdata.sh b/sphinx/scripts/specify_cloud_ec2_userdata.sh new file mode 100644 index 0000000..225170a --- /dev/null +++ b/sphinx/scripts/specify_cloud_ec2_userdata.sh @@ -0,0 +1,90 @@ +# Make sure to fill in all variables (starting with `$`) in the following userdata +# script before including it in an EC2 launch configuration. +# +# $BUCKET_NAME +# $DATABASE_NAME +# $DB_IDENTIFIER +# $REGION +# $MASTER_PASSWORD + +# Avoid services restarting during apt upgrade +sudo sed -i "s/#\$nrconf{kernelhints} = -1;/\$nrconf{kernelhints} = -1;/g" /etc/needrestart/needrestart.conf; +sudo sed -i "s/#\$nrconf{restart} = 'i';/\$nrconf{restart} = 'a';/g" /etc/needrestart/needrestart.conf; + +# Run apt installs +sudo apt update; +sudo apt upgrade -y; +sudo apt install -y apt-transport-https ca-certificates git gh curl software-properties-common wget python3-pip awscli mysql-client j2cli; + +# Configure AWS +aws configure set aws_access_key_id "ACCESS_KEY"; +aws configure set aws_secret_access_key "ACCESS_KEY_SECRET"; +aws configure set default.region REGION; +aws configure set default.output json; + +# Install Docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg; +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null; +sudo apt update; +apt-cache policy docker-ce; +sudo apt install -y docker-ce; +docker --version; + +# Install docker compose +mkdir .docker; +mkdir .docker/cli-plugins; +curl -L "https://github.com/docker/compose/releases/download/v2.17.2/docker-compose-linux-$(uname -m)" -o ~/.docker/cli-plugins/docker-compose; +chmod +x ~/.docker/cli-plugins/docker-compose; +docker compose version; + +# Python setup +# python3 -m pip install j2cli; +# export PATH=$PATH:/home/ubuntu/.local/bin; +# sudo apt install j2cli; + +# Copy files from S3 specify cloud bucket +aws s3 cp s3://$BUCKET_NAME/repo-snapshots/docker-compositions/ ./ --recursive; +aws s3 cp s3://$BUCKET_NAME/repo-snapshots/spcloudservers.json ./specifycloud/; +aws s3 cp s3://$BUCKET_NAME/repo-snapshots/defaults.env ./specifycloud/; + +# Database setup +mkdir seed-databases; +aws s3 cp s3://$BUCKET_NAME/seed-database/specify.sql ./seed-databases/; +mysql --host $DATABASE_NAME.$DB_IDENTIFIER.$REGION.rds.amazonaws.com --port 3306 \ + -u master -p$MASTER_PASSWORD -e "create database specify;"; +mysql --host $DATABASE_NAME.$DB_IDENTIFIER.$REGION.rds.amazonaws.com --port 3306 \ + -u master -p$MASTER_PASSWORD specify < ./seed-databases/specify.sql; +rm -f ./seed-databases/specify.sql; + +# Configure Specify Network +cd specifycloud; +touch spcloudservers.json; +touch defaults.env; + +# Run Specify Network +make; +sudo docker compose up -d; + +# Certbot setup +sudo apt install certbot python3-certbot-apache; +sudo mkdir /var/www/sp7demofish; + +# Github clone private repo +ssh-keygen -t ed25519 -C "your_email@address.com"; +#ssh-keygen -t rsa -b 4096 -C "your_email@address.com"; + +# git clone repos +git clone https://github.com/specify/specify7.git; +git clone https://github.com/specify/specify6.git; +git clone https://github.com/specify/report-runner-service.git; +#git clone https://github.com/specify/web-asset-server.git; + +# Install nginx +sudo apt install -y nginx openjdk-8-jdk maven ant; +sudo ufw allow 'Nginx HTTP'; +sudo ufw status; +sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-arm64/jre/bin/java; + +# Build without docker +cd specify6; +ant compile-nonmac; diff --git a/sphinx/scripts/specify_cloud_setup.sh b/sphinx/scripts/specify_cloud_setup.sh new file mode 100644 index 0000000..38c5c36 --- /dev/null +++ b/sphinx/scripts/specify_cloud_setup.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +sudo apt-get update; +sudo apt upgrade; +sudo apt install -y make python3-pip + +#sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose mysql-client; +#sudo apt install docker.io + +# Install Docker +sudo apt install -y apt-transport-https ca-certificates curl software-properties-common; +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg; +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null; +sudo apt update; +sudo apt -y install docker-ce; +docker --version; +sudo systemctl status docker; + +#sudo systemctl start docker.service +#sudo systemctl enable docker.service +#sudo systemctl status docker.service + +# Install docker compose +mkdir .docker +mkdir .docker/cli-plugins +curl -L "https://github.com/docker/compose/releases/download/v2.17.2/docker-compose-linux-$(uname -m)" -o ~/.docker/cli-plugins/docker-compose; +chmod +x ~/.docker/cli-plugins/docker-compose; +docker compose version; + +# Install awscli +sudo apt -y install awscli; +aws configure; # See aws credentials at bottom of notes + +# Setup mysql datbase connection +sudo apt install -y mysql-client; + +# Copy files from S3 +aws s3 cp s3://specify-cloud/repo-snapshots/docker-compositions/ ./ --recursive + +# python setup +sudo apt install -y python3-pip; +python3 -m pip install j2cli; + +# Create config files for specifycloud +cd specifycloud; +vim spcloudservers.json +vim defaults.env +sudo apt install j2cli +make + +# ssh client-server alive settings +sudo echo -e "ClientAliveInterval 1200\nClientAliveCountMax 3" >> /etc/ssh/sshd_config +sudo systemctl reload sshd; + +#docker pull specifyconsortium/specify7-service:edge + +# su specify -c make +docker-compose up -d