Split IPFS nodes into Server and GW, update Kubo

ethereum · Jul 14, 2023 · 9b373d5 · 9b373d5
1 parent 1ec64b1
commit 9b373d5
Show file tree

Hide file tree

Showing 10 changed files with 393 additions and 10 deletions.
diff --git a/environments/ipfs.yaml b/environments/ipfs.yaml
@@ -11,11 +11,12 @@ networks:
   source-verify:
 
 services:
-  ipfs:
-    image: ethereum/source-verify:ipfs-${TAG}
-    container_name: ipfs-${TAG}
+  ipfs-server:
+    build:
+      context: ../services/ipfs/server
+      dockerfile: Dockerfile.ipfs
+    container_name: ipfs-server-${TAG}
     ports:
-      - "${IPFS_GW_EXTERNAL_PORT}:8080"
       - "${IPFS_LIBP2P_EXTERNAL_PORT}:4001"
       - "${IPFS_API_EXTERNAL_PORT}:5001"
     env_file:
@@ -24,9 +25,17 @@ services:
     networks:
       - source-verify
     volumes:
-      - type: bind
-        source: $REPOSITORY_PATH
-        target: /repository
-      - type: bind
-        source: ../services/ipfs/ipfs-${TAG}.key
-        target: /sourcify/ipfs-${TAG}.key
+      - $REPOSITORY_PATH:/repository
+      - ../services/ipfs/server/ipfs-${TAG}.key:/sourcify/ipfs-${TAG}.key
+  ipfs-gateway:
+    image: ipfs/kubo:v0.21.0
+    container_name: ipfs-gateway-${TAG}
+    ports:
+      - "${IPFS_GW_EXTERNAL_PORT}:8080"
+    env_file:
+      - .env
+    restart: always
+    networks:
+      - source-verify
+    volumes:
+      - ../services/ipfs/gateway/:/container-init.d # Will run scripts here alpabetically
diff --git a/services/ipfs/gateway/001-gateway-init.sh b/services/ipfs/gateway/001-gateway-init.sh
@@ -0,0 +1,198 @@
+#!/bin/sh
+
+echo "Initializing IPFS gateway with custom settings"
+
+# Don't use the default subdomain gateway i.e. not <cid>.ipfs.localhost:8080 but localhost:8080/ipfs/<cid>
+ipfs config --json Gateway.PublicGateways '{"localhost": null }'
+
+# Add some content providers as peers for faster content discovery https://docs.ipfs.tech/how-to/peering-with-content-providers/#content-provider-list
+# Markdown converted to JSON with ChatGPT
+# web3.storage https://web3.storage/docs/reference/peering/
+ipfs config --json Peering.Peers '[
+ {
+    "ID": "QmcFf2FH3CEgTNHeMRGhN7HNHU1EXAxoEk6EFuSyXCsvRE",
+    "Addrs": [
+      "/dnsaddr/node-1.ingress.cloudflare-ipfs.com"
+    ]
+  },
+  {
+    "ID": "QmcFmLd5ySfk2WZuJ1mfSWLDjdmHZq7rSAua4GoeSQfs1z",
+    "Addrs": [
+      "/dnsaddr/node-2.ingress.cloudflare-ipfs.com"
+    ]
+  },
+  {
+    "ID": "QmcfFmzSDVbwexQ9Au2pt5YEXHK5xajwgaU6PpkbLWerMa",
+    "Addrs": [
+      "/dnsaddr/node-3.ingress.cloudflare-ipfs.com"
+    ]
+  },
+  {
+    "ID": "QmcfJeB3Js1FG7T8YaZATEiaHqNKVdQfybYYkbT1knUswx",
+    "Addrs": [
+      "/dnsaddr/node-4.ingress.cloudflare-ipfs.com"
+    ]
+  },
+  {
+    "ID": "QmcfVvzK4tMdFmpJjEKDUoqRgP4W9FnmJoziYX5GXJJ8eZ",
+    "Addrs": [
+      "/dnsaddr/node-5.ingress.cloudflare-ipfs.com"
+    ]
+  },
+  {
+    "ID": "QmcfZD3VKrUxyP9BbyUnZDpbqDnT7cQ4WjPP8TRLXaoE7G",
+    "Addrs": [
+      "/dnsaddr/node-6.ingress.cloudflare-ipfs.com"
+    ]
+  },
+  {
+    "ID": "QmcfZP2LuW4jxviTeG8fi28qjnZScACb8PEgHAc17ZEri3",
+    "Addrs": [
+      "/dnsaddr/node-7.ingress.cloudflare-ipfs.com"
+    ]
+  },
+  {
+    "ID": "QmcfgsJsMtx6qJb74akCw1M24X1zFwgGo11h1cuhwQjtJP",
+    "Addrs": [
+      "/dnsaddr/node-8.ingress.cloudflare-ipfs.com"
+    ]
+  },
+  {
+    "ID": "Qmcfr2FC7pFzJbTSDfYaSy1J8Uuy8ccGLeLyqJCKJvTHMi",
+    "Addrs": [
+      "/dnsaddr/node-9.ingress.cloudflare-ipfs.com"
+    ]
+  },
+  {
+    "ID": "QmcfR3V5YAtHBzxVACWCzXTt26SyEkxdwhGJ6875A8BuWx",
+    "Addrs": [
+      "/dnsaddr/node-10.ingress.cloudflare-ipfs.com"
+    ]
+  },
+  {
+    "ID": "Qmcfuo1TM9uUiJp6dTbm915Rf1aTqm3a3dnmCdDQLHgvL5",
+    "Addrs": [
+      "/dnsaddr/node-11.ingress.cloudflare-ipfs.com"
+    ]
+  },
+  {
+    "ID": "QmcfV2sg9zaq7UUHVCGuSvT2M2rnLBAPsiE79vVyK3Cuev",
+    "Addrs": [
+      "/dnsaddr/node-12.ingress.cloudflare-ipfs.com"
+    ]
+  },
+  {
+    "ID": "12D3KooWGaHbxpDWn4JVYud899Wcpa4iHPa3AMYydfxQDb3MhDME",
+    "Addrs": [
+      "/dnsaddr/ipfs.ssi.eecc.de"
+    ]
+  },
+  {
+    "ID": "12D3KooWCVXs8P7iq6ao4XhfAmKWrEeuKFWCJgqe9jGDMTqHYBjw",
+    "Addrs": [
+      "/ip4/139.178.68.217/tcp/6744"
+    ]
+  },
+  {
+    "ID": "12D3KooWGBWx9gyUFTVQcKMTenQMSyE2ad9m7c9fpjS4NMjoDien",
+    "Addrs": [
+      "/ip4/147.75.49.71/tcp/6745"
+    ]
+  },
+  {
+    "ID": "12D3KooWFrnuj5o3tx4fGD2ZVJRyDqTdzGnU3XYXmBbWbc8Hs8Nd",
+    "Addrs": [
+      "/ip4/147.75.86.255/tcp/6745"
+    ]
+  },
+  {
+    "ID": "12D3KooWN8vAoGd6eurUSidcpLYguQiGZwt4eVgDvbgaS7kiGTup",
+    "Addrs": [
+      "/ip4/3.134.223.177/tcp/6745"
+    ]
+  },
+  {
+    "ID": "12D3KooWLV128pddyvoG6NBvoZw7sSrgpMTPtjnpu3mSmENqhtL7",
+    "Addrs": [
+      "/ip4/35.74.45.12/udp/6746/quic"
+    ]
+  },
+  {
+    "ID": "QmWaik1eJcGHq1ybTWe7sezRfqKNcDRNkeBaLnGwQJz1Cj",
+    "Addrs": [
+      "/dnsaddr/fra1-1.hostnodes.pinata.cloud"
+    ]
+  },
+  {
+    "ID": "QmNfpLrQQZr5Ns9FAJKpyzgnDL2GgC6xBug1yUZozKFgu4",
+    "Addrs": [
+      "/dnsaddr/fra1-2.hostnodes.pinata.cloud"
+    ]
+  },
+  {
+    "ID": "QmPo1ygpngghu5it8u4Mr3ym6SEU2Wp2wA66Z91Y1S1g29",
+    "Addrs": [
+      "/dnsaddr/fra1-3.hostnodes.pinata.cloud"
+    ]
+  },
+  {
+    "ID": "QmRjLSisUCHVpFa5ELVvX3qVPfdxajxWJEHs9kN3EcxAW6",
+    "Addrs": [
+      "/dnsaddr/nyc1-1.hostnodes.pinata.cloud"
+    ]
+  },
+  {
+    "ID": "QmPySsdmbczdZYBpbi2oq2WMJ8ErbfxtkG8Mo192UHkfGP",
+    "Addrs": [
+      "/dnsaddr/nyc1-2.hostnodes.pinata.cloud"
+    ]
+  },
+  {
+    "ID": "QmSarArpxemsPESa6FNkmuu9iSE1QWqPX2R3Aw6f5jq4D5",
+    "Addrs": [
+      "/dnsaddr/nyc1-3.hostnodes.pinata.cloud"
+    ]
+  },
+  {
+    "ID": "12D3KooWFFhc8fPYnQXdWBCowxSV21EFYin3rU27p3NVgSMjN41k",
+    "Addrs": [
+      "/ip4/5.161.92.43/tcp/4001"
+    ]
+  },
+  {
+    "ID": "12D3KooWSW4hoHmDXmY5rW7nCi9XmGTy3foFt72u86jNP53LTNBJ",
+    "Addrs": [
+      "/ip4/5.161.55.227/tcp/4001"
+    ]
+  },
+  {
+    "ID": "12D3KooWSDj6JM2JmoHwE9AUUwqAFUEg9ndd3pMA8aF2bkYckZfo",
+    "Addrs": [
+      "/ip4/5.161.92.36/tcp/4001"
+    ]
+  },
+  {
+    "ID": "QmR69wtWUMm1TWnmuD4JqC1TWLZcc8iR2KrTenfZZbiztd",
+    "Addrs": [
+      "/ip4/104.210.43.77"
+    ]
+  },
+  {
+    "ID": "12D3KooWGASC2jm3pmohEJXUhuStkxDitPgzvs4qMuFPaiD9x1BA",
+    "Addrs": [
+      "/ip4/78.46.108.24"
+    ]
+  },
+  {
+    "ID": "12D3KooWRbWZN3GvLf9CHmozq4vnTzDD4EEoiqtRJxg5FV6Gfjmm",
+    "Addrs": [
+      "/ip4/65.109.19.136"
+    ]
+  },
+  {
+    "ID": "12D3KooWQ85aSCFwFkByr5e3pUCQeuheVhobVxGSSs1DrRQHGv1t",
+    "Addrs": [
+      "/dnsaddr/node-1.ipfs.4everland.net"
+    ]
+  }
+]'
diff --git a/services/ipfs/server/Dockerfile.ipfs b/services/ipfs/server/Dockerfile.ipfs
@@ -0,0 +1,17 @@
+# Explicit setting for Mac ARM
+FROM --platform=linux/amd64 ubuntu:20.04 
+RUN apt-get update && apt-get install -y cron curl
+
+RUN mkdir /sourcify
+WORKDIR /sourcify
+
+RUN curl https://dist.ipfs.tech/kubo/v0.21.0/kubo_v0.21.0_linux-amd64.tar.gz > kubo.tar.gz && tar xvfz kubo.tar.gz && cd kubo && ./install.sh
+
+ADD publish.sh ./publish.sh
+ADD init-config.sh ./init-config.sh
+ADD entrypoint.sh ./entrypoint.sh
+ADD cron.job ./cron.job
+
+# Terminate and auto-restart container if ipfs daemon crashes
+HEALTHCHECK --interval=2m CMD curl http://localhost:8080/ipfs/bafybeifx7yeb55armcsxwwitkymga5xf53dxiarykms3ygqic223w5sk3m || ipfs shutdown && pkill entrypoint.sh
+ENTRYPOINT ./entrypoint.sh
diff --git a/services/ipfs/server/README.md b/services/ipfs/server/README.md
@@ -0,0 +1,12 @@
+When testing use the following docker run command to prevent reimporting the repository each time. Note that the first time is mandatory.
+
+Setting `--env DEBUG=true` will:
+
+- prevent adding all the repo to ipfs (using the existing one, set with `-v /path_to_local_ipfs_folder:/root/.ipfs`)
+- prevent using the private keys
+- prevent running cron
+- prevent using remote pinning services
+
+```
+docker run  -it --rm --name sourcify_ipfs --env DEBUG=true  -v /path_to_local_sourcify_repo:/repository/ -v /path_to_local_ipfs_folder:/root/.ipfs  -p 5001:5001 -p 8080:8080 -p 4001:4001 ipfs_sourcify
+```
diff --git a/services/ipfs/server/cron.job b/services/ipfs/server/cron.job
@@ -0,0 +1,4 @@
+SHELL=/bin/bash
+
+0 */6 * * * /sourcify/publish.sh >> /var/log/cron.log 2>&1
+# This extra line makes it a valid cron
diff --git a/services/ipfs/server/entrypoint.sh b/services/ipfs/server/entrypoint.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+
+if [ ! -f /root/.ipfs/config ]
+then
+    echo "No config found. Initializing..."
+    bash ./init-config.sh
+fi
+
+
+ipfs daemon --enable-pubsub-experiment --enable-namesys-pubsub &
+
+# Wait for the daemon to initialize
+echo "Sleeping 30 seconds"
+sleep 30
+echo "Sleeped 30 seconds"
+
+if [ -z "$DEBUG" ]
+then
+    date
+    echo "Starting ipfs add"
+    hash=$(ipfs add -Q -r /repository/contracts)
+    echo "Finished ipfs add! New ipfs hash: $hash"
+    date
+
+    # Remove the old /contracts in MFS
+    echo "Removing /contracts from MFS"
+    ipfs files rm -r /contracts
+    echo "Removed /contracts from MFS"
+
+    # cp the repo under MFS
+    echo "Copying $hash to MFS at /contracts"
+    ipfs files cp -p /ipfs/$hash /contracts
+    echo "Copied $hash to MFS at /contracts"
+fi
+
+
+bash ./publish.sh
+
+# Write the TAG var to /etc/environment so that the crontab can pick up the variable
+echo "TAG=$TAG" > /etc/environment
+
+if [ -z "$DEBUG" ]
+then
+    crontab cron.job
+    cron -f
+fi
+
+tail -f /dev/null
diff --git a/services/ipfs/server/init-config.sh b/services/ipfs/server/init-config.sh
@@ -0,0 +1,36 @@
+ipfs init --profile=badgerds
+ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/8080
+
+## Build announced address config according to https://docs.ipfs.io/how-to/configure-node/#addresses. Need to announce the public and local IPs in swarm manually since docker does not know these IPs.
+ANNOUNCED_ADDRESSES='['
+if test -n "$PUBLIC_IP"
+then 
+  ANNOUNCED_ADDRESSES=''$ANNOUNCED_ADDRESSES'"/ip4/'$PUBLIC_IP'/tcp/'$IPFS_LIBP2P_EXTERNAL_PORT'","/ip4/'$PUBLIC_IP'/udp/'$IPFS_LIBP2P_EXTERNAL_PORT'/quic"'
+fi 
+
+if test -n "$LOCAL_IP"
+then
+  if test -n "$PUBLIC_IP" # Add comma if there are addresses in the array already
+  then 
+    ANNOUNCED_ADDRESSES=$ANNOUNCED_ADDRESSES','
+  fi
+  ANNOUNCED_ADDRESSES=''$ANNOUNCED_ADDRESSES'"/ip4/'$LOCAL_IP'/tcp/'$IPFS_LIBP2P_EXTERNAL_PORT'","/ip4/'$LOCAL_IP'/udp/'$IPFS_LIBP2P_EXTERNAL_PORT'/quic"'
+fi
+
+ANNOUNCED_ADDRESSES=$ANNOUNCED_ADDRESSES']'
+
+ipfs config Addresses.Announce $ANNOUNCED_ADDRESSES --json
+ipfs config --json Routing.AcceleratedDHTClient true
+
+# Allow WebUI to be accesible from host
+ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '["*"]'
+ipfs config --json Addresses.API '["/ip4/0.0.0.0/tcp/5001"]'
+
+if [ -z "$DEBUG" ]
+then
+  # Add remote pinning services
+  ipfs pin remote service add estuary https://api.estuary.tech/pinning $ESTUARY_PINNING_SECRET
+  ipfs pin remote service add web3.storage https://api.web3.storage/ $WEB3_STORAGE_PINNING_SECRET
+
+  ipfs key import main /sourcify/ipfs-${TAG}.key 
+fi
diff --git a/services/ipfs/server/ipfs-latest.key.gpg b/services/ipfs/server/ipfs-latest.key.gpg
diff --git a/services/ipfs/server/ipfs-stable.key.gpg b/services/ipfs/server/ipfs-stable.key.gpg
@@ -0,0 +1 @@
+�	ی&G횔:�ҋl���?��~�(�8��Hiy�R;��a%�#i����c.9�Z�w"V^��j-i�x�l�j����6l\_H#:T�ݩ>���s���uλ��gnV���:Q��q�S��S}Z�$V=�f���u�����X
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		� ی&G횔:�ҋl��?��~�(�8��Hiy�R;��a%�#i��c.9�Z�w"V^��j-i�x�l�j��6l\_H#:T�ݩ>��s��uλ��gnV��:Q��q�S��S}Z�$V=�f��u��X
Expand Down