Merge remote-tracking branch 'origin/master' into jeremy-bench-on-con…

…tainer-author-noted
moondance-labs · Dec 12, 2024 · a4ba66f · a4ba66f
2 parents 162a7f8 + ebf3c86
commit a4ba66f
Show file tree

Hide file tree

Showing 29 changed files with 1,291 additions and 339 deletions.
diff --git a/.github/workflows/e2e-test-bridge.yml b/.github/workflows/e2e-test-bridge.yml
@@ -7,8 +7,8 @@ on:
     types: [completed]
   workflow_dispatch:
     inputs:
-      run_id:
-        description: "Run id from which artifacts are taken"
+      run_sha:
+        description: "Sha commit from which artifacts are taken"
         required: true
 
 jobs:
@@ -19,9 +19,9 @@ jobs:
               id: retrieve-run-id
               run: |
                   if [[ -n "${{ github.event_name == 'workflow_dispatch' }}" ]]; then
-                    echo "run_id=${{ github.event.inputs.run_id }}" >> $GITHUB_OUTPUT
+                    echo "run_sha=${{ github.event.inputs.run_sha }}" >> $GITHUB_OUTPUT
                   else
-                    echo "run_id=${{ github.event.workflow_run.id }}" >> $GITHUB_OUTPUT
+                    echo "run_sha=${{ github.event.workflow_run.head_sha }}" >> $GITHUB_OUTPUT
                   fi
             - name: Recognize sha ref
               id: sharef
@@ -93,13 +93,18 @@ jobs:
               id: check_date
               run: |
                 date --version
+            - name: Check output
+              id: check_output
+              run: |
+                echo ${{ steps.retrieve-run-id.outputs.run_sha }}
             - name: Download build artifact from triggered workflow
-              uses: dawidd6/action-download-artifact@v2
+              uses: dawidd6/action-download-artifact@v7
               with:
-                run_id: ${{ steps.retrieve-run-id.outputs.run_id }}
+                commit: ${{ steps.retrieve-run-id.outputs.run_sha }}
+                workflow: release.yml
                 name: binaries
                 path: target/release
-                search_artifacts: true
+                allow_forks: false
             - name: "Make binaries executable"
               shell: bash
               run: |

diff --git a/.github/workflows/publish-docker-tannsi-relay.yml b/.github/workflows/publish-docker-tannsi-relay.yml
@@ -33,8 +33,16 @@ jobs:
           mkdir -p build
           VERSION="${{ github.event.inputs.tag }}"
           wget "${{ env.BASE_URL }}/$VERSION/${{ matrix.image.file_name }}" -O build/${{ matrix.image.file_name }}
+          wget "${{ env.BASE_URL }}/$VERSION/${{ matrix.image.file_name }}-execute-worker" -O build/${{ matrix.image.file_name }}-execute-worker
+          wget "${{ env.BASE_URL }}/$VERSION/${{ matrix.image.file_name }}-prepare-worker" -O build/${{ matrix.image.file_name }}-prepare-worker
+
           wget "${{ env.BASE_URL }}/$VERSION/${{ matrix.image.file_name }}-skylake" -O build/${{ matrix.image.file_name }}-skylake
+          wget "${{ env.BASE_URL }}/$VERSION/${{ matrix.image.file_name }}-execute-worker-skylake" -O build/${{ matrix.image.file_name }}-execute-worker-skylake
+          wget "${{ env.BASE_URL }}/$VERSION/${{ matrix.image.file_name }}-prepare-worker-skylake" -O build/${{ matrix.image.file_name }}-prepare-worker-skylake
+
           wget "${{ env.BASE_URL }}/$VERSION/${{ matrix.image.file_name }}-znver3" -O build/${{ matrix.image.file_name }}-znver3
+          wget "${{ env.BASE_URL }}/$VERSION/${{ matrix.image.file_name }}-execute-worker-znver3" -O build/${{ matrix.image.file_name }}-execute-worker-znver3
+          wget "${{ env.BASE_URL }}/$VERSION/${{ matrix.image.file_name }}-prepare-worker-znver3" -O build/${{ matrix.image.file_name }}-prepare-worker-znver3
 
       - name: push to docker
         run: |

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -63,6 +63,7 @@ pallet-data-preservers-runtime-api = { path = "pallets/data-preservers/runtime-a
 pallet-external-validator-slashes = { path = "pallets/external-validator-slashes", default-features = false }
 pallet-external-validators = { path = "pallets/external-validators", default-features = false }
 pallet-external-validators-rewards = { path = "pallets/external-validators-rewards", default-features = false }
+pallet-external-validators-rewards-runtime-api = { path = "pallets/external-validators-rewards/runtime-api", default-features = false }
 pallet-inflation-rewards = { path = "pallets/inflation-rewards", default-features = false }
 pallet-initializer = { path = "pallets/initializer", default-features = false }
 pallet-invulnerables = { path = "pallets/invulnerables", default-features = false }
@@ -260,9 +261,11 @@ xcm-runtime-apis = { git = "https://github.com/moondance-labs/polkadot-sdk", bra
 
 # Bridges (wasm)
 alloy-sol-types = { version = "0.4.2", default-features = false }
+bridge-hub-common = { git = "https://github.com/moondance-labs/polkadot-sdk", branch = "tanssi-polkadot-stable2407", default-features = false }
 milagro-bls = { package = "snowbridge-milagro-bls", version = "1.5.4", default-features = false }
 snowbridge-beacon-primitives = { git = "https://github.com/moondance-labs/polkadot-sdk", branch = "tanssi-polkadot-stable2409", default-features = false }
 snowbridge-core = { git = "https://github.com/moondance-labs/polkadot-sdk", branch = "tanssi-polkadot-stable2409", default-features = false }
+snowbridge-outbound-queue-merkle-tree = { git = "https://github.com/moondance-labs/polkadot-sdk", branch = "tanssi-polkadot-stable2409", default-features = false }
 snowbridge-pallet-ethereum-client = { git = "https://github.com/moondance-labs/polkadot-sdk", branch = "tanssi-polkadot-stable2409", default-features = false }
 snowbridge-pallet-inbound-queue = { git = "https://github.com/moondance-labs/polkadot-sdk", branch = "tanssi-polkadot-stable2409", default-features = false }
 snowbridge-pallet-inbound-queue-fixtures = { git = "https://github.com/moondance-labs/polkadot-sdk", branch = "tanssi-polkadot-stable2409", default-features = false }

diff --git a/pallets/collator-assignment/src/benchmarking.rs b/pallets/collator-assignment/src/benchmarking.rs
@@ -68,7 +68,7 @@ mod benchmarks {
         let container_chains: Vec<_> = (0..y).map(|i| ParaId::from(2000 + i)).collect();
         let session_index = 0u32.into();
         T::ContainerChains::set_session_container_chains(session_index, &container_chains);
-        T::RemoveParaIdsWithNoCredits::make_valid_para_ids(&container_chains);
+        T::ParaIdAssignmentHooks::make_valid_para_ids(&container_chains);
         T::HostConfiguration::set_host_configuration(session_index);
 
         // Assign random collators to test worst case: when collators need to be checked against existing collators

diff --git a/pallets/collator-assignment/src/lib.rs b/pallets/collator-assignment/src/lib.rs
@@ -54,9 +54,9 @@ use {
     },
     sp_std::{collections::btree_set::BTreeSet, fmt::Debug, prelude::*, vec},
     tp_traits::{
-        CollatorAssignmentHook, CollatorAssignmentTip, GetContainerChainAuthor,
-        GetHostConfiguration, GetSessionContainerChains, ParaId, RemoveInvulnerables,
-        RemoveParaIdsWithNoCredits, ShouldRotateAllCollators, Slot,
+        CollatorAssignmentTip, GetContainerChainAuthor, GetHostConfiguration,
+        GetSessionContainerChains, ParaId, ParaIdAssignmentHooks, RemoveInvulnerables,
+        ShouldRotateAllCollators, Slot,
     },
 };
 pub use {dp_collator_assignment::AssignedCollators, pallet::*};
@@ -105,8 +105,7 @@ pub mod pallet {
         type ShouldRotateAllCollators: ShouldRotateAllCollators<Self::SessionIndex>;
         type GetRandomnessForNextBlock: GetRandomnessForNextBlock<BlockNumberFor<Self>>;
         type RemoveInvulnerables: RemoveInvulnerables<Self::AccountId>;
-        type RemoveParaIdsWithNoCredits: RemoveParaIdsWithNoCredits;
-        type CollatorAssignmentHook: CollatorAssignmentHook<BalanceOf<Self>>;
+        type ParaIdAssignmentHooks: ParaIdAssignmentHooks<BalanceOf<Self>, Self::AccountId>;
         type Currency: Currency<Self::AccountId>;
         type CollatorAssignmentTip: CollatorAssignmentTip<BalanceOf<Self>>;
         type ForceEmptyOrchestrator: Get<bool>;
@@ -309,16 +308,13 @@ pub mod pallet {
                 old_assigned.container_chains.keys().cloned().collect();
 
             // Remove the containerChains that do not have enough credits for block production
-            T::RemoveParaIdsWithNoCredits::remove_para_ids_with_no_credits(
+            T::ParaIdAssignmentHooks::pre_assignment(
                 &mut container_chain_ids,
                 &old_assigned_para_ids,
             );
             // TODO: parathreads should be treated a bit differently, they don't need to have the same amount of credits
             // as parathreads because they will not be producing blocks on every slot.
-            T::RemoveParaIdsWithNoCredits::remove_para_ids_with_no_credits(
-                &mut parathreads,
-                &old_assigned_para_ids,
-            );
+            T::ParaIdAssignmentHooks::pre_assignment(&mut parathreads, &old_assigned_para_ids);
 
             let mut shuffle_collators = None;
             // If the random_seed is all zeros, we don't shuffle the list of collators nor the list
@@ -467,51 +463,11 @@ pub mod pallet {
 
             // TODO: this probably is asking for a refactor
             // only apply the onCollatorAssignedHook if sufficient collators
-            for para_id in &container_chain_ids {
-                if !new_assigned
-                    .container_chains
-                    .get(para_id)
-                    .unwrap_or(&vec![])
-                    .is_empty()
-                {
-                    if let Err(e) = T::CollatorAssignmentHook::on_collators_assigned(
-                        *para_id,
-                        maybe_tip.as_ref(),
-                        false,
-                    ) {
-                        // On error remove para from assignment
-                        log::warn!(
-                            "CollatorAssignmentHook error! Removing para {} from assignment: {:?}",
-                            u32::from(*para_id),
-                            e
-                        );
-                        new_assigned.container_chains.remove(para_id);
-                    }
-                }
-            }
-
-            for para_id in &parathreads {
-                if !new_assigned
-                    .container_chains
-                    .get(para_id)
-                    .unwrap_or(&vec![])
-                    .is_empty()
-                {
-                    if let Err(e) = T::CollatorAssignmentHook::on_collators_assigned(
-                        *para_id,
-                        maybe_tip.as_ref(),
-                        true,
-                    ) {
-                        // On error remove para from assignment
-                        log::warn!(
-                            "CollatorAssignmentHook error! Removing para {} from assignment: {:?}",
-                            u32::from(*para_id),
-                            e
-                        );
-                        new_assigned.container_chains.remove(para_id);
-                    }
-                }
-            }
+            T::ParaIdAssignmentHooks::post_assignment(
+                &old_assigned_para_ids,
+                &mut new_assigned.container_chains,
+                &maybe_tip,
+            );
 
             Self::store_collator_fullness(
                 &new_assigned,

diff --git a/pallets/collator-assignment/src/mock.rs b/pallets/collator-assignment/src/mock.rs
@@ -33,8 +33,8 @@ use {
     },
     sp_std::collections::{btree_map::BTreeMap, btree_set::BTreeSet},
     tp_traits::{
-        CollatorAssignmentHook, CollatorAssignmentTip, ParaId, ParathreadParams,
-        RemoveInvulnerables, RemoveParaIdsWithNoCredits, SessionContainerChains,
+        CollatorAssignmentTip, ParaId, ParaIdAssignmentHooks, ParathreadParams,
+        RemoveInvulnerables, SessionContainerChains,
     },
     tracing_subscriber::{layer::SubscriberExt, FmtSubscriber},
 };
@@ -310,23 +310,6 @@ impl CollatorAssignmentTip<u32> for MockCollatorAssignmentTip {
         }
     }
 }
-pub struct MockCollatorAssignmentHook;
-
-impl CollatorAssignmentHook<u32> for MockCollatorAssignmentHook {
-    fn on_collators_assigned(
-        para_id: ParaId,
-        _maybe_tip: Option<&u32>,
-        _is_parathread: bool,
-    ) -> Result<Weight, sp_runtime::DispatchError> {
-        // Only fail for para 1001
-        if MockData::mock().assignment_hook_errors && para_id == 1001.into() {
-            // The error doesn't matter
-            Err(sp_runtime::DispatchError::Unavailable)
-        } else {
-            Ok(Weight::default())
-        }
-    }
-}
 
 pub struct GetCoreAllocationConfigurationImpl;
 
@@ -352,8 +335,7 @@ impl pallet_collator_assignment::Config for Test {
         RotateCollatorsEveryNSessions<MockCollatorRotationSessionPeriod>;
     type GetRandomnessForNextBlock = MockGetRandomnessForNextBlock;
     type RemoveInvulnerables = RemoveAccountIdsAbove100;
-    type RemoveParaIdsWithNoCredits = RemoveParaIdsAbove5000;
-    type CollatorAssignmentHook = MockCollatorAssignmentHook;
+    type ParaIdAssignmentHooks = MockParaIdAssignmentHooksImpl;
     type CollatorAssignmentTip = MockCollatorAssignmentTip;
     type ForceEmptyOrchestrator = ConstBool<false>;
     type Currency = ();
@@ -414,16 +396,22 @@ impl RemoveInvulnerables<u64> for RemoveAccountIdsAbove100 {
 }
 
 /// Any ParaId >= 5000 will be considered to not have enough credits
-pub struct RemoveParaIdsAbove5000;
+pub struct MockParaIdAssignmentHooksImpl;
 
-impl RemoveParaIdsWithNoCredits for RemoveParaIdsAbove5000 {
-    fn remove_para_ids_with_no_credits(
-        para_ids: &mut Vec<ParaId>,
-        _currently_assigned: &BTreeSet<ParaId>,
-    ) {
+impl<AC> ParaIdAssignmentHooks<u32, AC> for MockParaIdAssignmentHooksImpl {
+    fn pre_assignment(para_ids: &mut Vec<ParaId>, _old_assigned: &BTreeSet<ParaId>) {
         para_ids.retain(|para_id| *para_id <= ParaId::from(5000));
     }
 
+    fn post_assignment(
+        _current_assigned: &BTreeSet<ParaId>,
+        new_assigned: &mut BTreeMap<ParaId, Vec<AC>>,
+        _maybe_tip: &Option<u32>,
+    ) -> Weight {
+        new_assigned.retain(|para_id, _| *para_id <= ParaId::from(5000));
+        Weight::zero()
+    }
+
     #[cfg(feature = "runtime-benchmarks")]
     fn make_valid_para_ids(_para_ids: &[ParaId]) {}
 }

diff --git a/pallets/collator-assignment/src/tests.rs b/pallets/collator-assignment/src/tests.rs
@@ -1344,64 +1344,6 @@ fn assign_collators_prioritizing_tip() {
     });
 }
 
-#[test]
-fn on_collators_assigned_hook_failure_removes_para_from_assignment() {
-    new_test_ext().execute_with(|| {
-        run_to_block(1);
-
-        MockData::mutate(|m| {
-            m.collators_per_container = 2;
-            m.collators_per_parathread = 2;
-            m.min_orchestrator_chain_collators = 5;
-            m.max_orchestrator_chain_collators = 5;
-
-            m.collators = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
-            m.container_chains = vec![1001, 1002, 1003, 1004];
-            m.assignment_hook_errors = false;
-        });
-        run_to_block(11);
-
-        assert_eq!(
-            assigned_collators(),
-            BTreeMap::from_iter(vec![
-                (1, 1000),
-                (2, 1000),
-                (3, 1000),
-                (4, 1000),
-                (5, 1000),
-                (6, 1001),
-                (7, 1001),
-                (8, 1002),
-                (9, 1002),
-                (10, 1003),
-                (11, 1003),
-            ]),
-        );
-
-        // Para 1001 will fail on_assignment_hook
-        MockData::mutate(|m| {
-            m.assignment_hook_errors = true;
-        });
-
-        run_to_block(21);
-
-        assert_eq!(
-            assigned_collators(),
-            BTreeMap::from_iter(vec![
-                (1, 1000),
-                (2, 1000),
-                (3, 1000),
-                (4, 1000),
-                (5, 1000),
-                (8, 1002),
-                (9, 1002),
-                (10, 1003),
-                (11, 1003),
-            ]),
-        );
-    });
-}
-
 #[test]
 fn assign_collators_truncates_before_shuffling() {
     // Check that if there are more collators than needed, we only assign the first collators