Last changes from NeonLabs

.dockerignore

@@ -2,7 +2,6 @@ Dockerfile
 Dockerfile.*
 .*.swp
 
-.git
 .github
 .gitignore
 .gitmodules

.github/workflows/deploy.py

@@ -29,9 +29,10 @@
 
 DOCKER_USER = os.environ.get("DHUBU")
 DOCKER_PASSWORD = os.environ.get("DHUBP")
-IMAGE_NAME = 'neonlabsorg/evm_loader'
-SOLANA_NODE_VERSION = 'v1.14.20'
-SOLANA_BPF_VERSION = 'v1.14.20'
+IMAGE_NAME = os.environ.get("IMAGE_NAME")
+RUN_LINK_REPO = os.environ.get("RUN_LINK_REPO")
+SOLANA_NODE_VERSION = 'v1.16.17'
+SOLANA_BPF_VERSION = 'v1.16.17'
 
 VERSION_BRANCH_TEMPLATE = r"[vt]{1}\d{1,2}\.\d{1,2}\.x.*"
 docker_client = docker.APIClient()
@@ -105,7 +106,7 @@ def run_tests(github_sha):
     project_name = f"neon-evm-{github_sha}"
     stop_containers(project_name)
 
-    run_subprocess(f"docker-compose -p {project_name} -f ./evm_loader/docker-compose-ci.yml up -d")
+    run_subprocess(f"docker-compose -p {project_name} -f ./ci/docker-compose-ci.yml up -d")
     container_name = get_solana_container_name(project_name)
     click.echo("Start tests")
     exec_id = docker_client.exec_create(
@@ -126,6 +127,9 @@ def run_tests(github_sha):
         print("Part of tests are skipped")
 
     exec_status = docker_client.exec_inspect(exec_id['Id'])["ExitCode"]
+
+    run_subprocess(f"docker-compose -p {project_name} -f ./ci/docker-compose-ci.yml logs dk-neon-api")
+
     stop_containers(project_name)
 
     if tests_are_failed or exec_status == 1:
@@ -134,7 +138,7 @@ def run_tests(github_sha):
 
 def get_solana_container_name(project_name):
     data = subprocess.run(
-        f"docker-compose -p {project_name} -f ./evm_loader/docker-compose-ci.yml ps",
+        f"docker-compose -p {project_name} -f ./ci/docker-compose-ci.yml ps",
         shell=True, capture_output=True, text=True).stdout
     click.echo(data)
     pattern = rf'{project_name}_solana_[1-9]+'
@@ -144,7 +148,7 @@ def get_solana_container_name(project_name):
 
 
 def stop_containers(project_name):
-    run_subprocess(f"docker-compose -p {project_name} -f ./evm_loader/docker-compose-ci.yml down")
+    run_subprocess(f"docker-compose -p {project_name} -f ./ci/docker-compose-ci.yml down")
 
 
 @cli.command(name="trigger_proxy_action")
@@ -155,16 +159,22 @@ def stop_containers(project_name):
 @click.option('--token')
 @click.option('--is_draft')
 @click.option('--labels')
-def trigger_proxy_action(head_ref_branch, base_ref_branch, github_ref, github_sha, token, is_draft, labels):
+@click.option('--pr_url')
+@click.option('--pr_number')
+def trigger_proxy_action(head_ref_branch, base_ref_branch, github_ref, github_sha, token, is_draft, labels,
+                         pr_url, pr_number):
     is_develop_branch = github_ref in ['refs/heads/develop', 'refs/heads/master']
     is_tag_creating = 'refs/tags/' in github_ref
     is_version_branch = re.match(VERSION_BRANCH_TEMPLATE, github_ref.replace("refs/heads/", "")) is not None
-    is_FTS_labeled_not_draft = 'FullTestSuit' in labels and is_draft != "true"
+    is_FTS_labeled_not_draft = 'fullTestSuit' in labels and is_draft != "true"
+    is_extended_FTS_labeled_not_draft = 'extendedFullTestSuit' in labels and is_draft != "true"
 
-    if is_develop_branch or is_tag_creating or is_version_branch or is_FTS_labeled_not_draft:
-        full_test_suite = "true"
+    if is_extended_FTS_labeled_not_draft:
+        test_set = "extendedFullTestSuite"
+    elif is_develop_branch or is_tag_creating or is_version_branch or is_FTS_labeled_not_draft:
+        test_set = "fullTestSuite"
     else:
-        full_test_suite = "false"
+        test_set = "basic"
 
     github = GithubClient(token)
 
@@ -181,14 +191,16 @@ def trigger_proxy_action(head_ref_branch, base_ref_branch, github_ref, github_sh
         proxy_branch = 'develop'
     click.echo(f"Proxy branch: {proxy_branch}")
 
+    initial_pr = f"{pr_url}/{pr_number}/comments" if pr_number else ""
+
     runs_before = github.get_proxy_runs_list(proxy_branch)
     runs_count_before = github.get_proxy_runs_count(proxy_branch)
-    github.run_proxy_dispatches(proxy_branch, github_ref, github_sha, full_test_suite)
+    github.run_proxy_dispatches(proxy_branch, github_ref, github_sha, test_set, initial_pr)
     wait_condition(lambda: github.get_proxy_runs_count(proxy_branch) > runs_count_before)
 
     runs_after = github.get_proxy_runs_list(proxy_branch)
     proxy_run_id = list(set(runs_after) - set(runs_before))[0]
-    link = f"https://github.com/neonlabsorg/proxy-model.py/actions/runs/{proxy_run_id}"
+    link = f"https://github.com/{RUN_LINK_REPO}/actions/runs/{proxy_run_id}"
     click.echo(f"Proxy run link: {link}")
     click.echo("Waiting for completed status...")
     wait_condition(lambda: github.get_proxy_run_info(proxy_run_id)["status"] == "completed", timeout_sec=10800, delay=5)

.github/workflows/github_api_client.py

@@ -1,9 +1,10 @@
 import click
 import requests
+import os
 
 
 class GithubClient():
-    PROXY_ENDPOINT = "https://api.github.com/repos/neonlabsorg/proxy-model.py"
+    PROXY_ENDPOINT = os.environ.get("PROXY_ENDPOINT")
 
     def __init__(self, token):
         self.headers = {"Authorization": f"Bearer {token}",
@@ -22,11 +23,12 @@ def get_proxy_runs_count(self, proxy_branch):
             f"{self.PROXY_ENDPOINT}/actions/workflows/pipeline.yml/runs?branch={proxy_branch}", headers=self.headers)
         return int(response.json()["total_count"])
 
-    def run_proxy_dispatches(self, proxy_branch, github_ref, github_sha, full_test_suite):
+    def run_proxy_dispatches(self, proxy_branch, github_ref, github_sha, test_set, initial_pr):
         data = {"ref": proxy_branch,
-                "inputs": {"full_test_suite": full_test_suite,
+                "inputs": {"test_set": test_set,
                            "neon_evm_commit": github_sha,
-                           "neon_evm_branch": github_ref}
+                           "neon_evm_branch": github_ref,
+                           "initial_pr": initial_pr}
                 }
         response = requests.post(
             f"{self.PROXY_ENDPOINT}/actions/workflows/pipeline.yml/dispatches", json=data, headers=self.headers)

.github/workflows/pipeline.yml

@@ -13,6 +13,9 @@ on:
 env:
   DHUBU: ${{secrets.DHUBU}}
   DHUBP: ${{secrets.DHUBP}}
+  IMAGE_NAME: ${{vars.IMAGE_NAME}}
+  PROXY_ENDPOINT: ${{vars.PROXY_ENDPOINT}}
+  RUN_LINK_REPO: ${{vars.RUN_LINK_REPO}}
   BUILD_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
 
 concurrency:
@@ -64,7 +67,9 @@ jobs:
           --github_ref=${{ github.ref }} \
           --token=${{secrets.GHTOKEN }} \
           --is_draft=${{github.event.pull_request.draft}} \
-          --labels='${{ toJson(github.event.pull_request.labels.*.name) }}'
+          --labels='${{ toJson(github.event.pull_request.labels.*.name) }}' \
+          --pr_url="${{ github.api_url }}/repos/${{ github.repository }}/issues" \
+          --pr_number="${{ github.event.pull_request.number }}"
   finalize-image:
     runs-on: neon-evm-1
     needs:

Dockerfile

@@ -1,39 +1,39 @@
 ARG SOLANA_IMAGE
 # Install BPF SDK
-FROM solanalabs/rust:1.64.0 AS builder
+FROM solanalabs/rust:1.69.0 AS builder
 RUN cargo install rustfilt
 WORKDIR /opt
 ARG SOLANA_BPF_VERSION
 RUN sh -c "$(curl -sSfL https://release.solana.com/"${SOLANA_BPF_VERSION}"/install)" && \
-    /root/.local/share/solana/install/active_release/bin/sdk/bpf/scripts/install.sh
+    /root/.local/share/solana/install/active_release/bin/sdk/sbf/scripts/install.sh
 ENV PATH=/root/.local/share/solana/install/active_release/bin:/usr/local/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
 
 # Build evm_loader
 FROM builder AS evm-loader-builder
-COPY ./evm_loader/ /opt/evm_loader/
-WORKDIR /opt/evm_loader
+COPY . /opt/neon-evm/
+WORKDIR /opt/neon-evm/evm_loader
 ARG REVISION
 ENV NEON_REVISION=${REVISION}
 RUN cargo fmt --check && \
     cargo clippy --release && \
     cargo build --release && \
-    cargo build-sbf --arch bpf --features devnet && cp target/deploy/evm_loader.so target/deploy/evm_loader-devnet.so && \
-    cargo build-sbf --arch bpf --features testnet && cp target/deploy/evm_loader.so target/deploy/evm_loader-testnet.so && \
-    cargo build-sbf --arch bpf --features govertest && cp target/deploy/evm_loader.so target/deploy/evm_loader-govertest.so && \
-    cargo build-sbf --arch bpf --features govertest,emergency && cp target/deploy/evm_loader.so target/deploy/evm_loader-govertest-emergency.so && \
-    cargo build-sbf --arch bpf --features mainnet && cp target/deploy/evm_loader.so target/deploy/evm_loader-mainnet.so && \
-    cargo build-sbf --arch bpf --features mainnet,emergency && cp target/deploy/evm_loader.so target/deploy/evm_loader-mainnet-emergency.so && \
-    cargo build-sbf --arch bpf --features ci --dump
+    cargo build-bpf --features devnet && cp target/deploy/evm_loader.so target/deploy/evm_loader-devnet.so && \
+    cargo build-bpf --features testnet && cp target/deploy/evm_loader.so target/deploy/evm_loader-testnet.so && \
+    cargo build-bpf --features govertest && cp target/deploy/evm_loader.so target/deploy/evm_loader-govertest.so && \
+    cargo build-bpf --features govertest,emergency && cp target/deploy/evm_loader.so target/deploy/evm_loader-govertest-emergency.so && \
+    cargo build-bpf --features mainnet && cp target/deploy/evm_loader.so target/deploy/evm_loader-mainnet.so && \
+    cargo build-bpf --features mainnet,emergency && cp target/deploy/evm_loader.so target/deploy/evm_loader-mainnet-emergency.so && \
+    cargo build-bpf --features ci --dump
 
 # Build Solidity contracts
 FROM ethereum/solc:0.8.0 AS solc
 FROM ubuntu:20.04 AS contracts
 RUN apt-get update && \
     DEBIAN_FRONTEND=nontineractive apt-get -y install xxd && \
     rm -rf /var/lib/apt/lists/* /var/lib/apt/cache/*
-COPY evm_loader/tests/contracts/*.sol /opt/
-COPY evm_loader/solidity/*.sol /opt/
+COPY tests/contracts/*.sol /opt/
+COPY solidity/*.sol /opt/
 #COPY evm_loader/tests/test_solidity_precompiles.json /opt/
 COPY --from=solc /usr/bin/solc /usr/bin/solc
 WORKDIR /opt/
@@ -51,10 +51,10 @@ RUN apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get -y install vim less openssl ca-certificates curl python3 python3-pip parallel && \
     rm -rf /var/lib/apt/lists/*
 
-COPY evm_loader/tests/requirements.txt /tmp/
+COPY tests/requirements.txt /tmp/
 RUN pip3 install -r /tmp/requirements.txt
 
-COPY /evm_loader/solidity/ /opt/contracts/contracts/
+#COPY /evm_loader/solidity/ /opt/contracts/contracts/
 WORKDIR /opt
 
 COPY --from=solana \
@@ -73,26 +73,26 @@ RUN /opt/solana/bin/solana program dump metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518
 COPY evm_loader/solana-run-neon.sh \
      /opt/solana/bin/
 
-COPY --from=evm-loader-builder /opt/evm_loader/target/deploy/evm_loader*.so /opt/
-COPY --from=evm-loader-builder /opt/evm_loader/target/deploy/evm_loader-dump.txt /opt/
-COPY --from=evm-loader-builder /opt/evm_loader/target/release/neon-cli /opt/
-COPY --from=evm-loader-builder /opt/evm_loader/target/release/neon-api /opt/
+COPY --from=evm-loader-builder /opt/neon-evm/evm_loader/target/deploy/evm_loader*.so /opt/
+COPY --from=evm-loader-builder /opt/neon-evm/evm_loader/target/deploy/evm_loader-dump.txt /opt/
+COPY --from=evm-loader-builder /opt/neon-evm/evm_loader/target/release/neon-cli /opt/
+COPY --from=evm-loader-builder /opt/neon-evm/evm_loader/target/release/neon-api /opt/
 COPY --from=solana /usr/bin/spl-token /opt/spl-token
 COPY --from=contracts /opt/ /opt/solidity/
 COPY --from=contracts /usr/bin/solc /usr/bin/solc
-COPY evm_loader/wait-for-solana.sh \
-    evm_loader/wait-for-neon.sh \
-    evm_loader/deploy-evm.sh \
-    evm_loader/deploy-test.sh \
-    evm_loader/create-test-accounts.sh \
-    evm_loader/evm_loader-keypair.json \
+COPY ci/wait-for-solana.sh \
+    ci/wait-for-neon.sh \
+    ci/deploy-evm.sh \
+    ci/deploy-test.sh \
+    ci/create-test-accounts.sh \
+    ci/evm_loader-keypair.json \
     /opt/
 
-COPY evm_loader/operator-keypairs/ /opt/operator-keypairs
-COPY evm_loader/tests /opt/tests
-COPY evm_loader/operator-keypairs/id.json /root/.config/solana/id.json
-COPY evm_loader/operator-keypairs/id2.json /root/.config/solana/id2.json
-COPY evm_loader/keys/ /opt/keys
+COPY ci/operator-keypairs/ /opt/operator-keypairs
+COPY tests /opt/tests
+COPY ci/operator-keypairs/id.json /root/.config/solana/id.json
+COPY ci/operator-keypairs/id2.json /root/.config/solana/id2.json
+COPY ci/keys/ /opt/keys
 
-ENV CONTRACTS_DIR=/opt/solidity/
+#ENV CONTRACTS_DIR=/opt/solidity/
 ENV PATH=/opt/solana/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt

evm_loader/docker-compose-ci.yml → ci/docker-compose-ci.yml

@@ -8,9 +8,13 @@ services:
       - SOLANA_URL=http://solana:8899
     hostname: solana
     ports:
-       - 8899
+       - "8899"
     expose:
       - "8899"
+    ulimits:
+      nofile:
+        soft: 1048576
+        hard: 1048576
     entrypoint:
       /opt/solana/bin/solana-run-neon.sh
     networks:
@@ -30,16 +34,11 @@ services:
       NEON_CHAIN_ID: 111
       COMMITMENT: confirmed
       NEON_DB_CLICKHOUSE_URLS: "http://45.250.253.36:8123;http://45.250.253.38:8123"
-      NEON_DB_INDEXER_HOST: 45.250.253.32
-      NEON_DB_INDEXER_PORT: 5432
-      NEON_DB_INDEXER_DATABASE: indexer
-      NEON_DB_INDEXER_USER: postgres
-      NEON_DB_INDEXER_PASSWORD: ""
       KEYPAIR: /opt/operator-keypairs/id.json
       FEEPAIR: /opt/operator-keypairs/id.json
     image: ${EVM_LOADER_IMAGE}
     ports:
-    - 8085
+    - "8085"
     networks:
       - net