fixed bug where nothing was save if the quorum had an empty tick data…

…; added send_signed_tx rpc
qubic · Mar 6, 2024 · 6dc4aa9 · 6dc4aa9
1 parent dfca681
commit 6dc4aa9
Show file tree

Hide file tree

Showing 13 changed files with 580 additions and 115 deletions.
diff --git a/README.md b/README.md
@@ -1,14 +1,133 @@
-## Docker usage:
-When running the docker container, you need to specify the following environment variables:
-- `QUBIC_ARCHIVER_QUBIC_NODE_IP` - the IP address of the Qubic node that archiver will connect to
-- `QUBIC_ARCHIVER_QUBIC_FALLBACK_TICK` - the start tick for the archiver to start archiving from (needs to be from current epoch)
+## High level description:
+The archive system consists of two services:
+- `qubic-archiver` - the archiver processor and HTTP server that provides rpc endpoints to query the archiver
+- `qubic-node-fetcher` - a service that is starting from a reliable node, gather the peers by "walking" from peer to peer and then filters them out so they don't have more than 30 ticks behind. This service is also exposing an http server for the qubic-archiver to retrieve the reliable peers
+
+## IMPORTANT
+Before starting the system, open the `docker-compose.yml` file and make sure that there is a reliable peer as a starting point for the node fetcher. It's defined in the `NODE_FETCHER_QUBIC_STARTING_PEER_IP` environment variable.
+
+## Run with docker-compose:
+```bash
+$ docker-compose up -d
+```
+
+## Available endpoints:
+
+### GetTickTransactions:
+```bash
+$ curl -sX POST  http://127.0.0.1:8000/qubic.archiver.archive.pb.ArchiveService/GetTickTransactions -d '{"tickNumber": 12795663}'
+{
+  "transactions": [
+    {
+      "sourceId": "PJUMDQZEAEDZNFAXATGGYGJULKGALYFCBYUAZRGAHGPXCDBDGEJOMKOFSKKD",
+      "destId": "AFZPUAIYVPNUYGJRQVLUKOPPVLHAZQTGLYAAUUNBXFTVTAMSBKQBLEIEPCVJ",
+      "amount": "0",
+      "tickNumber": 12795663,
+      "inputType": 0,
+      "inputSize": 32,
+      "inputHex": "716c692d637564614dfd03e48d610cd450b590e835155b6b9f55f91c516428b4",
+      "signatureHex": "1e597cb8f834797a2b58e8178f7f18219db4797e803277632299ebdc222a4e5194037c29d8dd87c2f8cebdf8dfb30f34b175e37565f8b01d059dc71cc1342200",
+      "txId": "qbrdxsmlfggneflkbskpqwjeqhgdssfuabhfapywobqcilroobgvruseuerk"
+    }
+  ]
+}
+```
+### GetTickData
+```bash
+$ curl -sX POST  http://127.0.0.1:8000/qubic.archiver.archive.pb.ArchiveService/GetTickData -d '{"tickNumber": 12795663}'
+{
+  "tickData": {
+    "computorIndex": 335,
+    "epoch": 99,
+    "tickNumber": 12795663,
+    "timestamp": "1709731981000",
+    "varStruct": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
+    "timeLock": "Nd5Fn0wHnQmcar+qphSK5ucbEtg/gkY82ffWs4HBbkk=",
+    "transactionIds": [
+      "qbrdxsmlfggneflkbskpqwjeqhgdssfuabhfapywobqcilroobgvruseuerk"
+    ],
+    "contractFees": [],
+    "signatureHex": "4aef8b84c2b594331eae649722ed747fa01a69b37e38538a0e3257c8f739950c7c7433d22d6fe77231972d51927fb29b3b4cc6dc9b202e609d42e5eddafe1c00"
+  }
+}
+```
+
+### GetTransaction
+```bash
+$ curl -sX POST  http://127.0.0.1:8000/qubic.archiver.archive.pb.ArchiveService/GetTransaction -d '{"txId": "qbrdxsmlfggneflkbskpqwjeqhgdssfuabhfapywobqcilroobgvruseuerk"}'
+{
+  "transaction": {
+    "sourceId": "PJUMDQZEAEDZNFAXATGGYGJULKGALYFCBYUAZRGAHGPXCDBDGEJOMKOFSKKD",
+    "destId": "AFZPUAIYVPNUYGJRQVLUKOPPVLHAZQTGLYAAUUNBXFTVTAMSBKQBLEIEPCVJ",
+    "amount": "0",
+    "tickNumber": 12795663,
+    "inputType": 0,
+    "inputSize": 32,
+    "inputHex": "716c692d637564614dfd03e48d610cd450b590e835155b6b9f55f91c516428b4",
+    "signatureHex": "1e597cb8f834797a2b58e8178f7f18219db4797e803277632299ebdc222a4e5194037c29d8dd87c2f8cebdf8dfb30f34b175e37565f8b01d059dc71cc1342200",
+    "txId": "qbrdxsmlfggneflkbskpqwjeqhgdssfuabhfapywobqcilroobgvruseuerk"
+  }
+}
+```
+
+### GetQuorumTickData
+```bash
+$ curl -sX POST  http://127.0.0.1:8000/qubic.archiver.archive.pb.ArchiveService/GetQuorumTickData  -d '{"tickNumber": 12795663}'
+
+```
+
+### Get Computors
 ```bash
-$ docker run -p 8000:8000 -p 8001:8001 -e QUBIC_ARCHIVER_QUBIC_NODE_IP="212.51.150.253" -e QUBIC_ARCHIVER_QUBIC_FALLBACK_TICK=12543674 -v /root/store:/app/store --name qubic-archiver -d ghcr.io/qubic/qubic-archiver:latest
+$ curl -sX POST  http://127.0.0.1:8000/qubic.archiver.archive.pb.ArchiveService/GetComputors -d '{"epoch": 99}'
+{
+  "computors": {
+    "epoch": 99,
+    "identities": [
+      "9263520905f41451c102f3e4c9a2434490cd766ceaac5f03a5300f7b5a5e60c7",
+      "b23fc58a4371aedac687ef76c2ebc666ac554f1aee07351a9aa503b6df05ad92",
+      "0d6fa2d10c02f7cc33fec302bd77535d91e9c905785c6b0e7f03ef7547279174",
+      "e245860121add2513340b0c47df7010410120a41368dcdb5246b581f2a63d99f",
+      "27402616c685f579e233e64056ad5553fd475541439e68ff6587da0f9ee05e8b",
+      "8931ddf58887c72eb693ab866212b620d7db79ba1ec1456826d67e313c7e50c5"
+    ],
+    "signatureHex": "77d93167d5cf9ba7aca8139fcbdb0624eaf6884d7a3bb79832dc5f59131f84da55ee9e29f81cd07fae746b4b7aa055b734c58ce21c3443aef5da521903d10d00"
+  }
+}
+```
+### GetIdentityInfo
+```bash
+$ curl -sX POST  http://127.0.0.1:8000/qubic.archiver.archive.pb.ArchiveService/GetIdentityInfo -d '{"identity": "PJUMDQZEAEDZNFAXATGGYGJULKGALYFCBYUAZRGAHGPXCDBDGEJOMKOFSKKD"}'
+{
+  "identityInfo": {
+    "id": "PJUMDQZEAEDZNFAXATGGYGJULKGALYFCBYUAZRGAHGPXCDBDGEJOMKOFSKKD",
+    "tickNumber": 12797233,
+    "balance": "1479289941",
+    "incomingAmount": "4835212306297",
+    "outgoingAmount": "4833733016356",
+    "nrIncomingTransfers": 826438,
+    "nrOutgoingTransfers": 47187,
+    "latestIncomingTransferTick": 12790393,
+    "latestOutgoingTransferTick": 12797113,
+    "siblingsHex": [
+      "2a5af1c66af3ef4a294e09f27aed030d3faeffb9a1910012468b9c7f3e46bd9f",
+      "705f57f0c8f11be888bb1e4d935a7582ca65e8212207404bc135b22a6e9bf450",
+      "c11fb66d62103d9a2e47b39b205354517d7014e15b6985343daec01f396da1d5",
+      "c6b45f22943acd48520886ccd104a580d85b41f39e803c29a8d2eeb0b0e62865"
+    ]
+  }
+}
 ```
 
-## Querying the archiver
+### GetLastProcessedTick
 ```bash
-$ curl -sX POST  http://127.0.0.1:8000/qubic.archiver.archive.pb.ArchiveService/GetTickTransactions -d '{"tickNumber": 12570631}'
-$ curl -sX POST  http://127.0.0.1:8000/qubic.archiver.archive.pb.ArchiveService/GetTickData -d '{"tickNumber": 12570631}'
-$ curl -sX POST  http://127.0.0.1:8000/qubic.archiver.archive.pb.ArchiveService/GetTransaction -d '{"txId": "qbizhhehgiijddcgbzdxqtkqcwuedcvgnrorvovdkavytqsukxlucyjhjxeg"}'
-```
+$ curl -sX POST  http://127.0.0.1:8000/qubic.archiver.archive.pb.ArchiveService/GetLastProcessedTick
+{
+  "lastProcessedTick": 12797164,
+  "lastProcessedTicksPerEpoch": {
+    "99": "12797164"
+  }
+} 
+```
+
+
+
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,43 @@
+version: '3'
+
+services:
+  qubic-archiver:
+    image: ghcr.io/qubic/qubic-archiver:v0.1.12
+    container_name: qubic-archiver
+    ports:
+      - "8000:8000"
+      - "8001:8001"
+    links:
+      - "qubic-node-fetcher"
+    depends_on:
+      - "qubic-node-fetcher"
+    environment:
+      QUBIC_ARCHIVER_SERVER_HTTP_HOST: "0.0.0.0:8000"
+      QUBIC_ARCHIVER_SERVER_GRPC_HOST: "0.0.0.0:8001"
+      QUBIC_ARCHIVER_POOL_NODE_FETCHER_URL: "http://qubic-node-fetcher:8080/peers"
+    volumes:
+      - ./store/archiver:/app/store
+    networks:
+      - qubic
+    restart: always
+
+  qubic-node-fetcher:
+    image: ghcr.io/qubic/go-node-fetcher:v0.1.1
+    container_name: qubic-node-fetcher
+    ports:
+      - "8080:8080"
+    environment:
+      NODE_FETCHER_QUBIC_STARTING_PEER_IP: "212.51.150.253"
+    volumes:
+      - ./store/node-fetcher:/app/store
+    networks:
+      - qubic
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://127.0.0.1:8080/peers" ]
+      interval: 30s
+      timeout: 5s
+      retries: 5
+    restart: always
+
+networks:
+  qubic:
diff --git a/main.go b/main.go
@@ -43,9 +43,10 @@ func run() error {
 			IdleTimeout        time.Duration `conf:"default:15s"`
 		}
 		Qubic struct {
-			NodePort           string        `conf:"default:21841"`
-			StorageFolder      string        `conf:"default:store"`
-			ProcessTickTimeout time.Duration `conf:"default:5s"`
+			NodePort             string        `conf:"default:21841"`
+			StorageFolder        string        `conf:"default:store"`
+			ProcessTickTimeout   time.Duration `conf:"default:5s"`
+			NrPeersToBroadcastTx int           `conf:"default:2"`
 		}
 	}
 
@@ -98,7 +99,7 @@ func run() error {
 		return errors.Wrap(err, "creating new connection pool")
 	}
 
-	rpcServer := rpc.NewServer(cfg.Server.GrpcHost, cfg.Server.HttpHost, ps, chPool)
+	rpcServer := rpc.NewServer(cfg.Server.GrpcHost, cfg.Server.HttpHost, ps, chPool, cfg.Qubic.NrPeersToBroadcastTx)
 	rpcServer.Start()
 
 	shutdown := make(chan os.Signal, 1)

diff --git a/processor/processor.go b/processor/processor.go
@@ -108,16 +108,21 @@ func (p *Processor) processOneByOne() error {
 func (p *Processor) getNextProcessingTick(ctx context.Context, currentTickInfo types.TickInfo) (uint64, error) {
 	lastTick, err := p.ps.GetLastProcessedTick(ctx)
 	if err != nil {
+		//handles first run of the archiver where there is nothing in storage
+		// in this case we start from the initial tick of the current epoch
 		if errors.Is(err, store.ErrNotFound) {
 			return uint64(currentTickInfo.InitialTick), nil
 		}
 
 		return 0, errors.Wrap(err, "getting last processed tick")
 	}
 
+	//handles the case where the initial tick of epoch returned by the node is greater than the last processed tick
+	// which means that we are in the next epoch and we should start from the initial tick of the current epoch
 	if uint64(currentTickInfo.InitialTick) > lastTick {
 		return uint64(currentTickInfo.InitialTick), nil
 	}
 
+	// otherwise we are in the same epoch and we should start from the last processed tick + 1
 	return lastTick + 1, nil
 }