Merge branch 'master' into fix/reduce-candidates-timeout

.github/workflows/test.yml

@@ -338,6 +338,7 @@ jobs:
           GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \
           GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \
           BUILDER_IMAGE_TAG=${{ env.BUILDER_LATEST_TAG }} \
+          GIT_REF_NAME=${{ env.GIT_REF_NAME }} \
           BUILD_TYPE=${{ matrix.options.build_type }}
 
       - name: "Push Kagome APT Package"

README.md

@@ -121,6 +121,33 @@ make debug_docker
 make clear
 ```
 
+### Installation from APT Package
+
+To install KAGOME releases using the provided package repository, follow these steps (tested on Ubuntu 24.04.1 LTS (Noble Numbat)):
+
+Update your package lists and install necessary utilities:
+
+```sh
+apt update && apt install -y gpg curl
+```
+
+Add the repository’s GPG signing key:
+
+```sh
+curl -fsSL https://europe-north1-apt.pkg.dev/doc/repo-signing-key.gpg | gpg --dearmor -o /usr/share/keyrings/europe-north-1-apt-archive-keyring.gpg
+```
+
+Add the KAGOME package repository to your sources list:
+```sh
+echo "deb [signed-by=/usr/share/keyrings/europe-north-1-apt-archive-keyring.gpg] https://europe-north1-apt.pkg.dev/projects/kagome-408211 kagome main" > /etc/apt/sources.list.d/kagome.list
+```
+
+Update the package lists and install KAGOME:
+
+```sh
+apt update && apt install -y kagome
+```
+
 ### Using KAGOME
 
 #### Obtaining database snapshot (optional)
@@ -180,15 +207,73 @@ kagome --validator --chain localchain.json --base-path base_path
 
 This command executes a KAGOME full node with an authority role.
 
+#### Running KAGOME as a Service
+
+You can run KAGOME as a service using a systemd service file. Below is an example of a service file to launch KAGOME Kusama validator:
+```sh
+[Unit]
+Description=Kagome Node
+
+[Service]
+User=kagome
+Group=kagome
+LimitCORE=infinity
+LimitNOFILE=65536
+ExecStart=kagome \ # should be in path
+  --name kagome-validator \
+  --base-path /home/kagome/dev/kagome-fun/kusama-node-1 \
+  --public-addr=/ip4/212.11.12.32/tcp/30334 \ # Address should be publicly accessible
+  --validator \
+  --listen-addr=/ip4/0.0.0.0/tcp/30334 \
+  --chain kusama \
+  --prometheus-port=9615 \
+  --prometheus-external \
+  --wasm-execution Compiled \
+  --telemetry-url 'wss://telemetry.polkadot.io/submit/ 1' \
+  --rpc-port=9944 \
+  --node-key 63808171009b35fc218f207442e355b0634561c84e0aec2093e3515113475624
+
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
+```
+
+##### Adding the Service File
+
+1. Copy the service file content into a new file named kagome.service.
+2. Move the file to the systemd directory:
+
+`sudo mv kagome.service /etc/systemd/system/`
+
+##### Starting the Kagome Node
+
+To start the Kagome node using systemd:
+1. Reload the systemd manager configuration:
+```sh
+sudo systemctl daemon-reload
+```
+
+2. (Optionally) Enable the Kagome service to start on boot:
+```sh
+sudo systemctl enable kagome
+```
+
+3. Start the Kagome service:
+```sh
+sudo systemctl start kagome
+```
+
+4. Check the status of the Kagome service:
+```sh
+sudo systemctl status kagome
+```
+
 #### Run KAGOME with collator
 
 Read [this](./examples/adder-collator) tutorial
 
-### Configuration Details
-To run a KAGOME node, you need to provide it with a genesis config, cryptographic keys, and a place to store db files.
-* Example of a genesis config file can be found in `examples/first_kagome_chain/localchain.json`
-* Example of a base path dir can be found in `examples/first_kagome_chain/base_path`
-* To create database files, just provide any base path into `kagome` executable (mind that start with authority role requires keys to start).
 
 
 ## Contributing Guides

core/parachain/approval/approval_distribution.cpp

@@ -3402,7 +3402,11 @@ namespace kagome::parachain {
           libp2p::SharedFn{[&, promise{std::move(promise)}]() mutable {
             promise.set_value(approvedAncestor(min, max));
           }});
-      return future.get();
+      try {
+        return future.get();
+      } catch (std::future_error &) {
+        return block_tree_->getLastFinalized();
+      }
     }
 
     if (max.number <= min.number) {

get_version.sh

@@ -5,6 +5,10 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+sanitize_version() {
+  echo "$1" | sed -E 's/[^a-zA-Z0-9.+~:-]/-/g'
+}
+
 realpath() {
   if [ -d "$1" ]; then
     cd "$1" && pwd
@@ -13,26 +17,31 @@ realpath() {
   fi
 }
 
-cd $(dirname "$(realpath "$0")")
+cd "$(dirname "$(realpath "$0")")"
+
+SANITIZED=false
+if [ "$#" -gt 0 ] && [ "$1" = "--sanitized" ]; then
+  SANITIZED=true
+fi
 
 if [ -x "$(which git 2>/dev/null)" ] && [ -e ".git" ]; then
   if [ -x "$(which sed 2>/dev/null)" ]; then
     HEAD=$(git rev-parse --short HEAD)
     COMMON=$(git merge-base HEAD master)
 
-    DESCR=$(git describe --tags --long ${COMMON})
+    DESCR=$(git describe --tags --long "${COMMON}")
     if [ "$DESCR" = "" ]; then
       DESCR=$HEAD-0-g$HEAD
     fi
 
-    TAG_IN_MASTER=$(echo $DESCR | sed -E "s/v?(.*)-([0-9]+)-g[a-f0-9]+/\1/")
-    TAG_TO_FORK_DISTANCE=$(echo $DESCR | sed -E "s/v?(.*)-([0-9]+)-g[a-f0-9]+/\2/")
+    TAG_IN_MASTER=$(echo "$DESCR" | sed -E "s/v?(.*)-([0-9]+)-g[a-f0-9]+/\1/")
+    TAG_TO_FORK_DISTANCE=$(echo "$DESCR" | sed -E "s/v?(.*)-([0-9]+)-g[a-f0-9]+/\2/")
 
     BRANCH=$(git branch --show-current)
     if [ "$BRANCH" = "" ]; then
       BRANCH=$HEAD
     fi
-    FORK_TO_HEAD_DISTANCE=$(git rev-list --count ${COMMON}..HEAD)
+    FORK_TO_HEAD_DISTANCE=$(git rev-list --count "${COMMON}..HEAD")
 
     RESULT=$TAG_IN_MASTER
     if [ "$TAG_TO_FORK_DISTANCE" != "0" ]; then
@@ -52,4 +61,8 @@ else
   RESULT="Unknown(no git)"
 fi
 
-echo $RESULT
+if [ "$SANITIZED" = true ]; then
+  RESULT=$(sanitize_version "$RESULT")
+fi
+
+echo "$RESULT"

housekeeping/docker/kagome-dev/.gitignore

@@ -1,6 +1,8 @@
 *-versions.txt
+*_version.txt
 commit_hash.txt
 
+
 cumulus
 polkadot
 

housekeeping/docker/kagome-dev/Makefile

@@ -19,15 +19,6 @@ PACKAGE_ARCHITECTURE ?= amd64
 # Debug, Release, RelWithDebInfo
 BUILD_TYPE ?= Release
 
-# Generated versions
-OS_IMAGE ?= $(OS_IMAGE_NAME):$(OS_IMAGE_TAG)@sha256:$(OS_IMAGE_HASH)
-OS_IMAGE_TAG_WITH_HASH := $(OS_IMAGE_TAG)@sha256:$(OS_IMAGE_HASH)
-OS_IMAGE_SHORT_HASH := $(shell echo $(OS_IMAGE_HASH) | cut -c1-7)
-BUILDER_IMAGE_TAG ?= $(OS_IMAGE_SHORT_HASH)_rust$(RUST_VERSION)_gcc$(GCC_VERSION)_llvm$(LLVM_VERSION)
-BUILDER_LATEST_TAG ?= latest
-TESTER_LATEST_TAG ?= latest
-DOCKERHUB_BUILDER_PATH ?= $(DOCKERHUB_REGISTRY_PATH)_builder
-
 # kagome_dev_docker_build Variables
 BUILD_DIR ?= build
 CACHE_DIR := $(shell pwd)/../../../../kagome/$(BUILD_DIR)/cache
@@ -38,12 +29,23 @@ GITHUB_HUNTER_TOKEN ?=
 CTEST_OUTPUT_ON_FAILURE ?= 1
 WERROR ?= OFF
 
+# Generated versions
+OS_IMAGE ?= $(OS_IMAGE_NAME):$(OS_IMAGE_TAG)@sha256:$(OS_IMAGE_HASH)
+OS_IMAGE_TAG_WITH_HASH := $(OS_IMAGE_TAG)@sha256:$(OS_IMAGE_HASH)
+OS_IMAGE_SHORT_HASH := $(shell echo $(OS_IMAGE_HASH) | cut -c1-7)
+BUILDER_IMAGE_TAG ?= $(OS_IMAGE_SHORT_HASH)_rust$(RUST_VERSION)_gcc$(GCC_VERSION)_llvm$(LLVM_VERSION)
+BUILDER_LATEST_TAG ?= latest
+TESTER_LATEST_TAG ?= latest
+DOCKERHUB_BUILDER_PATH ?= $(DOCKERHUB_REGISTRY_PATH)_builder
+KAGOME_SANITIZED_VERSION = $(shell cd $(WORKING_DIR) && ./get_version.sh --sanitized)
+
 # kagome_runtime_cache and kagome_image_build Variables
 KAGOME_PACKAGE_VERSION ?=
 #KAGOME_RUNTIME_PACKAGE_VERSION ?=
 
 # upload_apt_package Variables
 ARTIFACTS_REPO ?= kagome-apt
+PUBLIC_ARTIFACTS_REPO ?= kagome
 REGION ?= europe-north1
 
 # CI Variables
@@ -55,11 +57,11 @@ export DOCKER_BUILDKIT=1
 # BUILDKIT_PROGRESS - auto, plain, tty, rawjson
 export BUILDKIT_PROGRESS=auto
 
-
 get_versions:
 	@echo "full_commit_hash: `git rev-parse HEAD`" | tee commit_hash.txt
 	@echo "short_commit_hash: `git rev-parse HEAD | head -c 7`" | tee -a commit_hash.txt
 	@echo "kagome_version: `cd $(WORKING_DIR) && ./get_version.sh`" | tee kagome_version.txt
+	@echo "kagome_sanitized_version: $(KAGOME_SANITIZED_VERSION)" | tee -a kagome_version.txt
 
 builder_image_tag:
 	@echo $(BUILDER_IMAGE_TAG)
@@ -133,8 +135,17 @@ kagome_dev_docker_build:
 			$(PACKAGE_ARCHITECTURE) \
 			kagome-dev \
 			/tmp/kagome \
-			'Kagome Dev Debian Package' \
-			'$(DEPENDENCIES)' ; \
+			'Kagome Dev Ubuntu Package' \
+			'$(DEPENDENCIES)' && \
+		if [ "$(IS_MAIN_OR_TAG)" = "true" ] && [ "$(GIT_REF_NAME)" != "master" ] && [ "$(BUILD_TYPE)" = "Release" ]; then \
+			./build_apt_package.sh \
+				\"$(KAGOME_SANITIZED_VERSION)-$(BUILD_TYPE)\" \
+				$(PACKAGE_ARCHITECTURE) \
+				kagome \
+				/tmp/kagome \
+				'Kagome Ubuntu Package' \
+				'$(DEPENDENCIES)' ; \
+		fi; \
 		" || DOCKER_EXEC_RESULT=$$? ; \
 	if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \
 		echo "Error: Docker exec failed with return code $$DOCKER_EXEC_RESULT"; \
@@ -244,7 +255,10 @@ upload_apt_package:
 	SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \
 	gcloud config set artifacts/repository $(ARTIFACTS_REPO); \
 	gcloud config set artifacts/location $(REGION); \
-	gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/kagome-dev_$$(date +'%y.%m.%d')-$${SHORT_COMMIT_HASH}-$(BUILD_TYPE)_$(PACKAGE_ARCHITECTURE).deb
+	gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/kagome-dev_$$(date +'%y.%m.%d')-$${SHORT_COMMIT_HASH}-$(BUILD_TYPE)_$(PACKAGE_ARCHITECTURE).deb ; \
+	if [ "$(IS_MAIN_OR_TAG)" = "true" ] && [ "$(GIT_REF_NAME)" != "master" ] && [ "$(BUILD_TYPE)" = "Release" ]; then \
+		gcloud artifacts apt upload $(PUBLIC_ARTIFACTS_REPO) --source=./pkg/kagome_$(KAGOME_SANITIZED_VERSION)-$(BUILD_TYPE)_$(PACKAGE_ARCHITECTURE).deb ; \
+	fi;
 
 runtime_cache:
 	CONTAINER_NAME=kagome_dev_runtime_cache_$$(openssl rand -hex 6); \
@@ -274,7 +288,7 @@ runtime_cache:
 			$(PACKAGE_ARCHITECTURE) \
 			kagome-dev-runtime \
 			/tmp/kagome_runtime \
-			'Kagome Runtime Dev Debian Package' \
+			'Kagome Runtime Dev Ubuntu Package' \
 			'kagome-dev' \
 			 /tmp/kagome/runtimes-cache/ ; \
 		" || DOCKER_EXEC_RESULT=$$? ; \

housekeeping/docker/kagome-dev/build_apt_package.sh

@@ -68,7 +68,7 @@ mkdir -p ./pkg/${DIR_NAME}${BINARY_INSTALL_DIR}
 log "Working directory: $(pwd)/pkg/"
 
 log "Copying artifacts..."
-mv -f ${ARTIFACTS_DIR}/* ./pkg/${DIR_NAME}${BINARY_INSTALL_DIR}/
+cp -rf ${ARTIFACTS_DIR}/* ./pkg/${DIR_NAME}${BINARY_INSTALL_DIR}/
 
 log "Package: ${PACKAGE_NAME}"
 log "Version: ${VERSION}"

zombienet/docker/build_apt_package.sh

@@ -71,7 +71,7 @@ mkdir -p ./pkg/${DIR_NAME}/DEBIAN
 mkdir -p ./pkg/${DIR_NAME}/usr/local/bin
 
 log "Copying artifacts..."
-mv -f ${ARTIFACTS_DIR}/* ./pkg/${DIR_NAME}/usr/local/bin/
+cp -rf ${ARTIFACTS_DIR}/* ./pkg/${DIR_NAME}/usr/local/bin/
 
 log "Creating control file..."
 cat <<EOF > ./pkg/${DIR_NAME}/DEBIAN/control