From e45af12b2a00537d187b23ea32728623c67420df Mon Sep 17 00:00:00 2001 From: hdahme Date: Fri, 18 Nov 2022 18:10:34 -0800 Subject: [PATCH 1/2] + terraform config and gcloud setup scripts --- .gitignore | 13 +++- juno-arb/.env.copy | 1 + juno-arb/README.md | 8 +-- juno-arb/main.py | 3 +- juno-arb/ops/gcp/cloudrun/.dockerignore | 1 + juno-arb/ops/gcp/cloudrun/Dockerfile | 18 +++++ juno-arb/ops/gcp/cloudrun/Readme.md | 42 +++++++++++ juno-arb/ops/gcp/terraform/Readme.md | 72 +++++++++++++++++++ juno-arb/ops/gcp/terraform/backend.tf | 3 + juno-arb/ops/gcp/terraform/gce.tf | 30 ++++++++ juno-arb/ops/gcp/terraform/iam.tf | 22 ++++++ juno-arb/ops/gcp/terraform/outputs.tf | 3 + juno-arb/ops/gcp/terraform/provider.template | 4 ++ juno-arb/ops/gcp/terraform/variables.template | 29 ++++++++ juno-arb/ops/gcp/terraform/vpc.tf | 21 ++++++ juno-arb/requirements.txt | 2 - 16 files changed, 264 insertions(+), 8 deletions(-) create mode 100644 juno-arb/.env.copy create mode 100644 juno-arb/ops/gcp/cloudrun/.dockerignore create mode 100644 juno-arb/ops/gcp/cloudrun/Dockerfile create mode 100644 juno-arb/ops/gcp/cloudrun/Readme.md create mode 100644 juno-arb/ops/gcp/terraform/Readme.md create mode 100644 juno-arb/ops/gcp/terraform/backend.tf create mode 100644 juno-arb/ops/gcp/terraform/gce.tf create mode 100644 juno-arb/ops/gcp/terraform/iam.tf create mode 100644 juno-arb/ops/gcp/terraform/outputs.tf create mode 100644 juno-arb/ops/gcp/terraform/provider.template create mode 100644 juno-arb/ops/gcp/terraform/variables.template create mode 100644 juno-arb/ops/gcp/terraform/vpc.tf diff --git a/.gitignore b/.gitignore index 0a3ee40..eac37b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,14 @@ /venv/ /__pycache__/ -juno-arb/__pycache__/ \ No newline at end of file + +# juno-arb +juno-arb/__pycache__/ +juno-arb/.env +juno-arb/skip_response.log +juno-arb/ops/gcp/terraform/.terraform* + +juno-arb/ops/gcp/terraform/provider.tf +juno-arb/ops/gcp/terraform/variables.tf + +juno-arb/Dockerfile +juno-arb/.dockerignore \ No newline at end of file diff --git a/juno-arb/.env.copy b/juno-arb/.env.copy new file mode 100644 index 0000000..a9aaafc --- /dev/null +++ b/juno-arb/.env.copy @@ -0,0 +1 @@ +export MNEMONIC="" \ No newline at end of file diff --git a/juno-arb/README.md b/juno-arb/README.md index cc42b43..9d1a6da 100644 --- a/juno-arb/README.md +++ b/juno-arb/README.md @@ -15,10 +15,10 @@ Once you have python 3.10, install all the dependencies: pip install -r requirements.txt ``` -Then, edit the global variables in the main.py file to -your liking. The most important being your mnemonic. -```python -MNEMONIC = "" +Copy .env.copy and populate with your mnemonic. **Don't commit this** + +```bash +source .env ``` Lastly, run the bot: diff --git a/juno-arb/main.py b/juno-arb/main.py index 4292ce1..cdc4ade 100644 --- a/juno-arb/main.py +++ b/juno-arb/main.py @@ -1,4 +1,5 @@ # General Imports +import os import json import aiometer import asyncio @@ -34,7 +35,7 @@ # Mnenomic to generate private key # Replace with your own mnemonic -MNEMONIC = "" +MNEMONIC = os.environ.get("MNEMONIC") # RPC URL to stream mempool and query contract state from RPC_URL = "https://rpc-juno-ia.cosmosia.notional.ventures/" diff --git a/juno-arb/ops/gcp/cloudrun/.dockerignore b/juno-arb/ops/gcp/cloudrun/.dockerignore new file mode 100644 index 0000000..2eea525 --- /dev/null +++ b/juno-arb/ops/gcp/cloudrun/.dockerignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/juno-arb/ops/gcp/cloudrun/Dockerfile b/juno-arb/ops/gcp/cloudrun/Dockerfile new file mode 100644 index 0000000..613ad6b --- /dev/null +++ b/juno-arb/ops/gcp/cloudrun/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.10-slim + +RUN apt-get clean \ + && apt-get -y update + +RUN apt-get -y install \ + nginx \ + python3-dev \ + build-essential + +WORKDIR /app + +COPY requirements.txt /app/requirements.txt +RUN pip install -r requirements.txt --src /usr/local/src + +COPY . . + +CMD [ "python3", "main.py" ] \ No newline at end of file diff --git a/juno-arb/ops/gcp/cloudrun/Readme.md b/juno-arb/ops/gcp/cloudrun/Readme.md new file mode 100644 index 0000000..b7747b7 --- /dev/null +++ b/juno-arb/ops/gcp/cloudrun/Readme.md @@ -0,0 +1,42 @@ +Cloudrun is if you want the easiest, lowest overhead way of running a bot. If you want more flexibility and customizability, check out the `terraform` config. + +Since a job times out after a maximum of 1 hour, we get around that by scheduling this job to run every hour, at minute 0 + +Because these commands are fairly symmetric with Cloud Build, you should be able to plug them into your CI/CD process with little/no lift + +Firstly, copy the `.dockerignore` and `Dockerfile` to the root of the bot. Then submit the build to cloudrun + +```bash +export REGION="us-central1" +export PROJECT=$(gcloud config get-value project) +export PROJECT_NUMBER=`gcloud projects list | grep $PROJECT | awk '{print $(NF)}'` +export IMAGE="us-docker.pkg.dev/$PROJECT/gcr.io/skip-mev/juno-arb:latest" +export JOB_NAME="skip-mev-juno-arb" +source .env + +# Build the image +gcloud builds submit --tag $IMAGE + +# Create the job +gcloud beta run jobs create $JOB_NAME \ + --image $IMAGE \ + --region $REGION \ + --task-timeout 1h \ + --set-env-vars MNEMONIC="$MNEMONIC" + +# Either run the job now +gcloud beta run jobs execute $JOB_NAME --region $REGION + +# Or schedule the job +gcloud scheduler jobs create http $JOB_NAME \ + --location $REGION \ + --schedule "0 * * * *" \ + --uri="https://$REGION-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/$PROJECT/jobs/$JOB_NAME:run" \ + --http-method POST \ + --oauth-service-account-email $PROJECT_NUMBER-compute@developer.gserviceaccount.com + +# To delete the job and cancel any scheduled executions +gcloud beta run jobs delete $JOB_NAME +``` + +You can monitor job executions at https://console.cloud.google.com/run and view logs at https://console.cloud.google.com/run/jobs/details/$REGION/$JOB_NAME/executions \ No newline at end of file diff --git a/juno-arb/ops/gcp/terraform/Readme.md b/juno-arb/ops/gcp/terraform/Readme.md new file mode 100644 index 0000000..03131c2 --- /dev/null +++ b/juno-arb/ops/gcp/terraform/Readme.md @@ -0,0 +1,72 @@ +If you want to use a persistent GCE instance to manage your bot, use this config. A persistent GCE instance offers more flexibility in line with more overhead, say if you want to run sidecar services and so on. Use the cloudrun config if you're just looking for the quick and dirty. + +Firstly, set up your environment and create the VM + +```bash +export TF_VAR_zone="us-central1-c" +export TF_VAR_instance_name="skip-mev-juno-arb" +export PROJECT=$(gcloud config get-value project) + +# Copy the .template files +cp variables.template variables.tf +cp provider.template provider.tf + +# Dynamic substitution for the terraform config +sed -i "" s/PROJECT_ID/$PROJECT/g variables.tf +sed -i "" s/PROJECT_ID/$PROJECT/g provider.tf +sed -i "" s/ZONE/$TF_VAR_zone/g variables.tf +sed -i "" s/ZONE/$TF_VAR_zone/g provider.tf +sed -i "" s/INSTANCE_NAME/$TF_VAR_instance_name/g variables.tf + +# You can always reset to the template with the following +# sed -i "" s/$PROJECT/PROJECT_ID/g variables.template +# sed -i "" s/$PROJECT/PROJECT_ID/g provider.template +# sed -i "" s/$TF_VAR_zone/ZONE/g variables.template +# sed -i "" s/$TF_VAR_zone/ZONE/g provider.template +# sed -i "" s/$TF_VAR_instance_name/INSTANCE_NAME/g variables.template + +# Initialize terraform state in GCS +terraform init \ + -backend-config="bucket=$PROJECT-tfstate" \ + -backend-config="prefix=$TF_VAR_instance_name" +terraform apply +``` + +Now, we can copy the bot to the instance. Make sure the mnemonic in your `.env` is popualted correctly. + +```bash +gcloud compute scp --recurse ../../../*.py ../../../*.json ../../../*.txt ../../../.env ubuntu@$TF_VAR_instance_name:/home/ubuntu --zone $TF_VAR_zone + +# SSH onto the machine and follow setup instructions for the bot, namely +gcloud compute ssh --zone $TF_VAR_zone ubuntu@$TF_VAR_instance_name --tunnel-through-iap --project $PROJECT + +# Set up dependencies to run from the root, so that the startup script can function +sudo apt update +sudo apt-get install python3-pip -y +sudo mkdir -p /app && sudo cp ~/.env ~/* /app +cd /app +sudo pip install -r requirements.txt +exit +``` + +[Optionally, but recommended] install the cloudwatch ops agent +*You can also install via the console, by visiting https://console.cloud.google.com/monitoring/dashboards/resourceList/gce_instance* + +```bash +cd ~/ && curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +sudo bash add-google-cloud-ops-agent-repo.sh --also-install +``` + +Finally, restart the instance. The bot is initialized via the startup script specified in the terraform config + +```bash +gcloud compute instances reset $TF_VAR_instance_name --zone $TF_VAR_zone +``` + +To verify the startup script worked, you can run `sudo journalctl -u google-startup-scripts.service` and can view logs by querying the following in the logs explorer + +```bash +resource.type="gce_instance" +sourceLocation.function="main.setupAndRunScript" +resource.labels.instance_id="THE_INSTANCE_ID_RETURNED_FROM_TERRAFORM" +``` \ No newline at end of file diff --git a/juno-arb/ops/gcp/terraform/backend.tf b/juno-arb/ops/gcp/terraform/backend.tf new file mode 100644 index 0000000..f174bec --- /dev/null +++ b/juno-arb/ops/gcp/terraform/backend.tf @@ -0,0 +1,3 @@ +terraform { + backend "gcs" {} +} \ No newline at end of file diff --git a/juno-arb/ops/gcp/terraform/gce.tf b/juno-arb/ops/gcp/terraform/gce.tf new file mode 100644 index 0000000..fdac2b3 --- /dev/null +++ b/juno-arb/ops/gcp/terraform/gce.tf @@ -0,0 +1,30 @@ +resource "google_compute_instance" "default" { + name = var.instance_name + machine_type = var.machine_type + zone = var.zone + tags = [var.instance_name, "ssh"] + project = var.project + + boot_disk { + initialize_params { + image = "ubuntu-os-cloud/ubuntu-2204-lts" + } + } + + metadata_startup_script = "cd /app && source .env && python3 main.py &" + + network_interface { + network = "default" + + access_config { + // Ephemeral public IP + } + } + + service_account { + email = google_service_account.default.email + scopes = ["cloud-platform"] + } + + allow_stopping_for_update = true +} diff --git a/juno-arb/ops/gcp/terraform/iam.tf b/juno-arb/ops/gcp/terraform/iam.tf new file mode 100644 index 0000000..a3a5ede --- /dev/null +++ b/juno-arb/ops/gcp/terraform/iam.tf @@ -0,0 +1,22 @@ +resource "google_service_account" "default" { + account_id = "${var.instance_name}-sa" + display_name = "Service Account" +} + +resource "google_project_iam_binding" "logs-writer-iam" { + role = "roles/logging.logWriter" + project = var.project + + members = [ + "serviceAccount:${google_service_account.default.email}", + ] +} + +resource "google_project_iam_binding" "metrics-writer-iam" { + role = "roles/monitoring.metricWriter" + project = var.project + + members = [ + "serviceAccount:${google_service_account.default.email}", + ] +} \ No newline at end of file diff --git a/juno-arb/ops/gcp/terraform/outputs.tf b/juno-arb/ops/gcp/terraform/outputs.tf new file mode 100644 index 0000000..e58820e --- /dev/null +++ b/juno-arb/ops/gcp/terraform/outputs.tf @@ -0,0 +1,3 @@ +output "instance_id" { + value = "${google_compute_instance.default.instance_id}" +} \ No newline at end of file diff --git a/juno-arb/ops/gcp/terraform/provider.template b/juno-arb/ops/gcp/terraform/provider.template new file mode 100644 index 0000000..61c202f --- /dev/null +++ b/juno-arb/ops/gcp/terraform/provider.template @@ -0,0 +1,4 @@ +provider "google" { + project = "PROJECT_ID" + zone = "ZONE" +} \ No newline at end of file diff --git a/juno-arb/ops/gcp/terraform/variables.template b/juno-arb/ops/gcp/terraform/variables.template new file mode 100644 index 0000000..ad992dd --- /dev/null +++ b/juno-arb/ops/gcp/terraform/variables.template @@ -0,0 +1,29 @@ +variable project { + type = string + default = "PROJECT_ID" + description = "Project ID" +} + +variable "zone" { + type = string + default = "ZONE" + description = "Zone" +} + +variable "region" { + type = string + default = "us-central1" + description = "Region" +} + +variable "instance_name" { + type = string + default = "INSTANCE_NAME" + description = "Instance Name" +} + +variable "machine_type" { + type = string + default = "e2-small" + description = "Machine Type" +} \ No newline at end of file diff --git a/juno-arb/ops/gcp/terraform/vpc.tf b/juno-arb/ops/gcp/terraform/vpc.tf new file mode 100644 index 0000000..251c01b --- /dev/null +++ b/juno-arb/ops/gcp/terraform/vpc.tf @@ -0,0 +1,21 @@ +resource "google_compute_network" "skip-mev" { + name = "skip-mev" + auto_create_subnetworks = false + mtu = 1460 + project = var.project + +} + +resource "google_compute_firewall" "ssh" { + name = "allow-ssh" + project = var.project + allow { + ports = ["22"] + protocol = "tcp" + } + direction = "INGRESS" + network = google_compute_network.skip-mev.id + priority = 1000 + source_ranges = ["0.0.0.0/0"] + target_tags = ["ssh"] +} \ No newline at end of file diff --git a/juno-arb/requirements.txt b/juno-arb/requirements.txt index 41898f0..2392411 100644 --- a/juno-arb/requirements.txt +++ b/juno-arb/requirements.txt @@ -28,7 +28,6 @@ google-api-python-client==2.65.0 google-auth==2.14.1 google-auth-httplib2==0.1.0 googleapis-common-protos==1.56.4 -grpcio==1.47.0 h11==0.12.0 httpcore==0.15.0 httplib2==0.21.0 @@ -48,7 +47,6 @@ pexpect==4.8.0 pickleshare==0.7.5 platformdirs==2.5.3 prompt-toolkit==3.0.32 -protobuf==3.20.3 psutil==5.9.4 ptyprocess==0.7.0 pure-eval==0.2.2 From 0ef4908d3d2b2f97681bebfa89af9fed572cb36b Mon Sep 17 00:00:00 2001 From: hdahme Date: Fri, 6 Jan 2023 10:19:25 -0800 Subject: [PATCH 2/2] Claim that keyerror bounty --- juno-arb/mempool.py | 18 ++++++++++++++---- juno-arb/ops/gcp/terraform/Readme.md | 9 ++++++++- juno-arb/requirements.txt | 2 +- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/juno-arb/mempool.py b/juno-arb/mempool.py index cb1e313..9166960 100644 --- a/juno-arb/mempool.py +++ b/juno-arb/mempool.py @@ -73,8 +73,13 @@ def check_for_swap_txs_in_mempool(rpc_url: str, already_seen: dict) -> list: backrun_potential_list.append(swap_tx) break except KeyError: - logging.error("KeyError, most likely a non-junoswap contract-swap message: ", message_value.contract) - continue + msg = f"KeyError, most likely a non-junoswap contract-swap message: {message_value.contract}" + try: + logging.info(msg) + except TypeError: + print(msg) + finally: + continue # If the message is a JunoSwap pass through swap elif 'pass_through_swap' in msg: # Create a PassThroughSwap object, append to the list @@ -91,8 +96,13 @@ def check_for_swap_txs_in_mempool(rpc_url: str, already_seen: dict) -> list: backrun_potential_list.append(pass_through_swap_tx) break except KeyError: - logging.error("KeyError, most likely a non-junoswap contract-pass_through_swap message: ", message_value.contract) - continue + msg = f"KeyError, most likely a non-junoswap contract-pass_through_swap message: {message_value.contract}" + try: + logging.info(msg) + except TypeError: + print(msg) + finally: + continue # If we found a tx with a swap message, return the list # to begin the process of checking for an arb opportunity if len(backrun_potential_list) > 0: diff --git a/juno-arb/ops/gcp/terraform/Readme.md b/juno-arb/ops/gcp/terraform/Readme.md index 03131c2..9bd36d4 100644 --- a/juno-arb/ops/gcp/terraform/Readme.md +++ b/juno-arb/ops/gcp/terraform/Readme.md @@ -69,4 +69,11 @@ To verify the startup script worked, you can run `sudo journalctl -u google-star resource.type="gce_instance" sourceLocation.function="main.setupAndRunScript" resource.labels.instance_id="THE_INSTANCE_ID_RETURNED_FROM_TERRAFORM" -``` \ No newline at end of file +``` + +For the easy appoach + +1. Connect via the ssh command above, and run `screen`. This opens a persistent session +2. python3 main.py +3. Press ctrl + a, ctrl + d Now you can disconnect from ssh and your code will continue to run. +4. You can reconnect to the screen via screen -r \ No newline at end of file diff --git a/juno-arb/requirements.txt b/juno-arb/requirements.txt index 2392411..f7d35cd 100644 --- a/juno-arb/requirements.txt +++ b/juno-arb/requirements.txt @@ -65,7 +65,7 @@ requests==2.28.1 rfc3986==1.5.0 rsa==4.9 six==1.16.0 -skip-python==0.1.1 +skip-python==0.1.2 sniffio==1.3.0 stack-data==0.6.0 tornado==6.2