Skip to content

Commit

Permalink
Add curvefi deployment
Browse files Browse the repository at this point in the history
  • Loading branch information
@hhio618
hhio618 committed Sep 13, 2023
1 parent e3a5f4d commit a9b328d
Show file tree
Hide file tree
Showing 3 changed files with 139 additions and 28 deletions.
10 changes: 10 additions & 0 deletions config/Curvefi.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"Coins": [
{
"Symbol": "Curve Coin",
"ForeignName": "CRV",
"Exchange": "Curvefi",
"Ignore": false
}
]
}
85 changes: 85 additions & 0 deletions deployments/k8s-yaml/scraper-dex-curve.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
apiVersion: "v1"
kind: Pod
metadata:
name: scraper-exchange-curve
spec:
containers:
- name: scraper-exchange-curve
image: dia.genericcollector.dev:latest
imagePullPolicy: Never
command: ["collector"]
args: ["-exchange=Curvefi", "-mode=current", "-pairsfile=true"]
env:
- name: USE_ENV
value: "true"
- name: POSTGRES_USER
value: "postgres"
- name: POSTGRES_PASSWORD
value: "password"
- name: POSTGRES_DB
value: "postgres"
- name: POSTGRES_HOST
value: "data-postgres.default.svc.cluster.local"
- name: INFLUXURL
value: "http://data-influx.default.svc.cluster.local:8086"
- name: INFLUXUSER
value: "test"
- name: INFLUXPASSWORD
value: "testtest"
- name: REDISURL
value: "data-redis.default.svc.cluster.local:6379"
- name: KAFKAURL
value: "data-kafka.default.svc.cluster.local:9094"
- name: ETHEREUM_URI_REST
value: "https://mainnet.infura.io/v3/fc2e181454424779b1f09e1f47304bc3"
- name: ETHEREUM_URI_WS
value: "wss://mainnet.infura.io/ws/v3/fc2e181454424779b1f09e1f47304bc3"
volumeMounts:
- name: env-context
mountPath: /mnt/env-context
readOnly: true
initContainers:
- name: check-postgres-connection
image: public.ecr.aws/docker/library/postgres:15-alpine
env:
- name: PGHOST
value: "data-postgres.default.svc.cluster.local"
- name: PGUSER
value: "postgres"
- name: PGPASSWORD
value: "password"
command:
["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"]
- name: check-redis-connection
image: public.ecr.aws/docker/library/redis:7
env:
- name: REDISCLI_URL
value: "redis://data-redis.default.svc.cluster.local:6379/0"
command:
["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"]
- name: check-influxdb-connection
image: public.ecr.aws/docker/library/influxdb:1.8
env:
- name: INFLUXDB_URL
value: "data-influx.default.svc.cluster.local"
- name: INFLUXDB_USERNAME
value: "test"
- name: INFLUXDB_PASSWORD
value: "testtest"
command:
[
"sh",
"-c",
"until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done",
]
- name: check-kafka-connection
image: confluentinc/cp-kafkacat:6.2.1
env:
- name: KAFKA_BROKER_URL
value: "data-kafka.default.svc.cluster.local:9094"
command:
["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"]
volumes:
- name: env-context
hostPath:
path: /mnt/diadata
72 changes: 44 additions & 28 deletions testenv.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ function main() {
esac
done


command=("$@")
# Check dependencies
if ! hash git 2> /dev/null; then echo "Git not found" >&2; return 1; fi
if ! hash docker 2> /dev/null; then echo "Docker not found" >&2; return 1; fi
Expand All @@ -203,8 +203,8 @@ function main() {
# Variables
local minikube_profile=dia
local minikube_k8s_version=v1.25.7
local minikube_hw_cpus=4
local minikube_hw_ram=8g
local minikube_hw_cpus=2
local minikube_hw_ram=5g
local minikube_hw_disk=50g
local minikube_driver=docker
local snapshot_docker_registry=https://registry.hub.docker.com/v2/
Expand All @@ -226,7 +226,7 @@ function main() {
local version_detected
version_detected=$(git describe --tags --abbrev=0)
declare -a demos_scraper_cex=("bitfinex" "bittrex" "coinbase" "mexc")
declare -a demos_scraper_dex=("platypus" "orca")
declare -a demos_scraper_dex=("platypus" "orca", "curve")
declare -a demos_scraper_liquidity=("platypus" "orca")
declare -a demos_scraper_foreign=("yahoofinance")

Expand Down Expand Up @@ -290,8 +290,8 @@ function main() {
_build_ifnotexist Dockerfile-tradesBlockService-Dev dia.tradesblockservice.dev
_build_ifnotexist Dockerfile-pairDiscoveryService-Dev dia.pairdiscoveryservice.dev
_build_ifnotexist Dockerfile-genericCollector-Dev dia.genericcollector.dev
_build_ifnotexist Dockerfile-genericForeignScraper-Dev dia.genericforeignscraper.dev
_build_ifnotexist Dockerfile-liquidityScraper-Dev dia.liquidityscraper.dev
# _build_ifnotexist Dockerfile-genericForeignScraper-Dev dia.genericforeignscraper.dev
# _build_ifnotexist Dockerfile-liquidityScraper-Dev dia.liquidityscraper.dev
_build_ifnotexist Dockerfile-assetCollectionService-Dev dia.assetcollectionservice.dev
if [ "$arg_full_mode" = true ]; then
_build_ifnotexist Dockerfile-blockchainservice-Dev dia.blockchainservice.dev
Expand All @@ -311,8 +311,8 @@ function main() {
_image_exist dia.tradesblockservice.dev || exit 1
_image_exist dia.pairdiscoveryservice.dev || exit 1
_image_exist dia.genericcollector.dev || exit 1
_image_exist dia.genericforeignscraper.dev || exit 1
_image_exist dia.liquidityscraper.dev || exit 1
# _image_exist dia.genericforeignscraper.dev || exit 1
# _image_exist dia.liquidityscraper.dev || exit 1
_image_exist dia.assetcollectionservice.dev || exit 1
if [ "$arg_full_mode" = true ]; then
_image_exist dia.blockchainservice.dev || exit 1
Expand Down Expand Up @@ -382,32 +382,48 @@ function main() {
fi

# Ask for variables with _EXTRACT and export them
read -p "Enter Postgres Server: " PGHOST_EXTRACT
read -p "Enter Postgres Port: " PGPORT_EXTRACT
read -p "Enter Postgres User: " PGUSER_EXTRACT
read -p "Enter Postgres Password: " PGPASSWORD_EXTRACT
read -p "Enter Postgres Database: " PGDB_EXTRACT
# read -p "Enter Postgres Server: " PGHOST_EXTRACT
# read -p "Enter Postgres Port: " PGPORT_EXTRACT
# read -p "Enter Postgres User: " PGUSER_EXTRACT
# read -p "Enter Postgres Password: " PGPASSWORD_EXTRACT
# read -p "Enter Postgres Database: " PGDB_EXTRACT

# export PGHOST=${PGHOST_EXTRACT}
# export PGUSER=${PGUSER_EXTRACT}
# export PGDB=${PGDB_EXTRACT}
# export PGPASSWORD=${PGPASSWORD_EXTRACT}
# export PGPORT=${PGPORT_EXTRACT}

export PGHOST=${PGHOST_EXTRACT}
export PGUSER=${PGUSER_EXTRACT}
export PGDB=${PGDB_EXTRACT}
export PGPASSWORD=${PGPASSWORD_EXTRACT}
export PGPORT=${PGPORT_EXTRACT}
export PGHOST=localhost
export PGUSER=postgres
export PGDB=postgres
export PGPASSWORD=password
export PGPORT=5432

# Download and unzip the file from rest.diadata.org
wget -qO snapshot.zip http://rest.diadata.org/snapshot/latest
unzip -o snapshot.zip
# wget -qO snapshot.zip http://rest.diadata.org/snapshot/latest
# unzip -o snapshot.zip

# Run the psql commands
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} --file output-blockchain.sql
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} --file output-exchange.sql
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} --file output-asset.sql
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} --file output-pool.sql
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} --file output-poolasset.sql
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} --file output-exchangepair.sql
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table blockchain cascade"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table asset cascade"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table exchange cascade"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table exchangepair cascade"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table pool cascade"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table poolasset cascade"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy asset FROM 'output_assets.csv' DELIMITER ';' CSV"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy blockchain FROM 'output_blockchain.csv' DELIMITER ';' CSV"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchangepair FROM 'output_exchangepair.csv' DELIMITER ';' CSV"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchange FROM 'output_cex.csv' DELIMITER ';' CSV"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchange FROM 'output_dex.csv' DELIMITER ';' CSV"
# FIXME: this will raise: ERROR: invalid input syntax for type numeric: "2023-09-07 09:05:45.012443"
# psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy poolasset FROM 'output_poolasset.csv' DELIMITER ';' CSV"
psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy pool FROM 'output_pool.csv' DELIMITER ';' CSV"


# Delete the output-*.sql files
rm output-blockchain.sql
# Delete the output-*.sql and snapshot.zip files
# rm output_*.csv
# rm snapshot.zip

echo "Data import and processing completed successfully."
fi
Expand Down

0 comments on commit a9b328d

Please sign in to comment.