feat: add da clients (#2743)

## What ❔

This PR changes the approach to managing 3rd party DA clients. It was
assumed before that they will be stored in a separate repository
(hyperchain-da), but to simplify the processes and solve a recursive
dependency problem, we decided to manage those within `zksync-era`.

The config now defines which DA client will be used, for proto-based
configuration it requires adding these lines to general.yaml:
```
da_client:
  avail:
    api_node_url: wss://turing-rpc.avail.so/ws
    bridge_api_url: undefined
    seed: SEED_PHRASE
    app_id: 82
    timeout: 3
    max_retries: 5
``` 
for env-based:
```
    DA_CLIENT="Avail"
    DA_API_NODE_URL="localhost:12345"
    DA_BRIDGE_API_URL="localhost:54321"
    DA_SEED="SEED_PHRASE"
    DA_APP_ID=1
    DA_TIMEOUT=2
    DA_MAX_RETRIES=3
```
If no config is provided - the default behavior is to use NoDA client
(same as now).

The `da_client` config might be merged with `da_dispatcher` at some
point as the second depends on the first one, so their separation does
not make much sense (apart from simplification of the configs). But I'd
prefer to do it as a separate PR in case we decide to merge them.

The client was reimplemented using only lightweight libraries from
crates.io, so it doesn't have any visible impact on build time.

## Why ❔

To enable seamless integration with 3rd party DA clients in
`zksync-era`.

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zk fmt` and `zk lint`.

Cargo.toml

@@ -38,6 +38,7 @@ members = [
     "core/node/base_token_adjuster",
     "core/node/external_proof_integration_api",
     "core/node/logs_bloom_backfill",
+    "core/node/da_clients",
     # Libraries
     "core/lib/db_connection",
     "core/lib/zksync_core_leftovers",
@@ -50,7 +51,6 @@ members = [
     "core/lib/dal",
     "core/lib/env_config",
     "core/lib/da_client",
-    "core/lib/default_da_clients",
     "core/lib/eth_client",
     "core/lib/eth_signer",
     "core/lib/l1_contract_interface",
@@ -196,6 +196,15 @@ trybuild = "1.0"
 vise = "0.2.0"
 vise-exporter = "0.2.0"
 
+# DA clients' dependencies
+# Avail
+base58 = "0.2.0"
+scale-encode = "0.5.0"
+blake2b_simd = "1.0.2"
+subxt-metadata = "0.34.0"
+parity-scale-codec = { version = "3.6.9", default-features = false }
+subxt-signer = { version = "0.34", default-features = false }
+
 # Here and below:
 # We *always* pin the latest version of protocol to disallow accidental changes in the execution logic.
 # However, for the historical version of protocol crates, we have lax requirements. Otherwise,
@@ -245,7 +254,6 @@ zksync_db_connection = { version = "0.1.0", path = "core/lib/db_connection" }
 zksync_env_config = { version = "0.1.0", path = "core/lib/env_config" }
 zksync_eth_client = { version = "0.1.0", path = "core/lib/eth_client" }
 zksync_da_client = { version = "0.1.0", path = "core/lib/da_client" }
-zksync_default_da_clients = { version = "0.1.0", path = "core/lib/default_da_clients" }
 zksync_eth_signer = { version = "0.1.0", path = "core/lib/eth_signer" }
 zksync_health_check = { version = "0.1.0", path = "core/lib/health_check" }
 zksync_l1_contract_interface = { version = "0.1.0", path = "core/lib/l1_contract_interface" }
@@ -279,6 +287,7 @@ zksync_commitment_generator = { version = "0.1.0", path = "core/node/commitment_
 zksync_house_keeper = { version = "0.1.0", path = "core/node/house_keeper" }
 zksync_node_genesis = { version = "0.1.0", path = "core/node/genesis" }
 zksync_da_dispatcher = { version = "0.1.0", path = "core/node/da_dispatcher" }
+zksync_da_clients = { version = "0.1.0", path = "core/node/da_clients" }
 zksync_eth_sender = { version = "0.1.0", path = "core/node/eth_sender" }
 zksync_node_db_pruner = { version = "0.1.0", path = "core/node/db_pruner" }
 zksync_node_fee_model = { version = "0.1.0", path = "core/node/fee_model" }

core/bin/zksync_server/Cargo.toml

@@ -21,7 +21,7 @@ zksync_utils.workspace = true
 zksync_types.workspace = true
 zksync_core_leftovers.workspace = true
 zksync_node_genesis.workspace = true
-zksync_default_da_clients.workspace = true
+zksync_da_clients.workspace = true
 
 # Consensus dependenices
 zksync_consensus_crypto.workspace = true

core/bin/zksync_server/src/main.rs

@@ -17,9 +17,9 @@ use zksync_config::{
         L1Secrets, ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig,
         ProtectiveReadsWriterConfig, Secrets,
     },
-    ApiConfig, BaseTokenAdjusterConfig, ContractVerifierConfig, DADispatcherConfig, DBConfig,
-    EthConfig, EthWatchConfig, ExternalProofIntegrationApiConfig, GasAdjusterConfig, GenesisConfig,
-    ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig,
+    ApiConfig, BaseTokenAdjusterConfig, ContractVerifierConfig, DAClientConfig, DADispatcherConfig,
+    DBConfig, EthConfig, EthWatchConfig, ExternalProofIntegrationApiConfig, GasAdjusterConfig,
+    GenesisConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig,
 };
 use zksync_core_leftovers::{
     temp_config_store::{decode_yaml_repr, TempConfigStore},
@@ -199,6 +199,7 @@ fn load_env_config() -> anyhow::Result<TempConfigStore> {
         gas_adjuster_config: GasAdjusterConfig::from_env().ok(),
         observability: ObservabilityConfig::from_env().ok(),
         snapshot_creator: SnapshotsCreatorConfig::from_env().ok(),
+        da_client_config: DAClientConfig::from_env().ok(),
         da_dispatcher_config: DADispatcherConfig::from_env().ok(),
         protective_reads_writer_config: ProtectiveReadsWriterConfig::from_env().ok(),
         basic_witness_input_producer_config: BasicWitnessInputProducerConfig::from_env().ok(),

core/bin/zksync_server/src/node_builder.rs

@@ -3,14 +3,13 @@
 
 use anyhow::Context;
 use zksync_config::{
-    configs::{eth_sender::PubdataSendingMode, wallets::Wallets, GeneralConfig, Secrets},
+    configs::{
+        da_client::DAClient, eth_sender::PubdataSendingMode, wallets::Wallets, GeneralConfig,
+        Secrets,
+    },
     ContractsConfig, GenesisConfig,
 };
 use zksync_core_leftovers::Component;
-use zksync_default_da_clients::{
-    no_da::wiring_layer::NoDAClientWiringLayer,
-    object_store::{config::DAObjectStoreConfig, wiring_layer::ObjectStorageClientWiringLayer},
-};
 use zksync_metadata_calculator::MetadataCalculatorConfig;
 use zksync_node_api_server::{
     tx_sender::{ApiContracts, TxSenderConfig},
@@ -28,6 +27,10 @@ use zksync_node_framework::{
         commitment_generator::CommitmentGeneratorLayer,
         consensus::MainNodeConsensusLayer,
         contract_verification_api::ContractVerificationApiLayer,
+        da_clients::{
+            avail::AvailWiringLayer, no_da::NoDAClientWiringLayer,
+            object_store::ObjectStorageClientWiringLayer,
+        },
         da_dispatcher::DataAvailabilityDispatcherLayer,
         eth_sender::{EthTxAggregatorLayer, EthTxManagerLayer},
         eth_watch::EthWatchLayer,
@@ -500,16 +503,23 @@ impl MainNodeBuilder {
         Ok(self)
     }
 
-    fn add_no_da_client_layer(mut self) -> anyhow::Result<Self> {
-        self.node.add_layer(NoDAClientWiringLayer);
-        Ok(self)
-    }
+    fn add_da_client_layer(mut self) -> anyhow::Result<Self> {
+        let Some(da_client_config) = self.configs.da_client_config.clone() else {
+            tracing::warn!("No config for DA client, using the NoDA client");
+            self.node.add_layer(NoDAClientWiringLayer);
+            return Ok(self);
+        };
+
+        match da_client_config.client {
+            DAClient::Avail(config) => {
+                self.node.add_layer(AvailWiringLayer::new(config));
+            }
+            DAClient::ObjectStore(config) => {
+                self.node
+                    .add_layer(ObjectStorageClientWiringLayer::new(config));
+            }
+        }
 
-    #[allow(dead_code)]
-    fn add_object_storage_da_client_layer(mut self) -> anyhow::Result<Self> {
-        let object_store_config = DAObjectStoreConfig::from_env()?;
-        self.node
-            .add_layer(ObjectStorageClientWiringLayer::new(object_store_config.0));
         Ok(self)
     }
 
@@ -750,7 +760,7 @@ impl MainNodeBuilder {
                     self = self.add_commitment_generator_layer()?;
                 }
                 Component::DADispatcher => {
-                    self = self.add_no_da_client_layer()?.add_da_dispatcher_layer()?;
+                    self = self.add_da_client_layer()?.add_da_dispatcher_layer()?;
                 }
                 Component::VmRunnerProtectiveReads => {
                     self = self.add_vm_runner_protective_reads_layer()?;

core/lib/config/src/configs/da_client/avail.rs

@@ -0,0 +1,11 @@
+use serde::Deserialize;
+
+#[derive(Clone, Debug, PartialEq, Deserialize)]
+pub struct AvailConfig {
+    pub api_node_url: String,
+    pub bridge_api_url: String,
+    pub seed: String,
+    pub app_id: u32,
+    pub timeout: usize,
+    pub max_retries: usize,
+}

core/lib/config/src/configs/da_client/mod.rs

@@ -0,0 +1,20 @@
+use serde::Deserialize;
+
+use crate::{AvailConfig, ObjectStoreConfig};
+
+pub mod avail;
+
+pub const AVAIL_CLIENT_CONFIG_NAME: &str = "Avail";
+pub const OBJECT_STORE_CLIENT_CONFIG_NAME: &str = "ObjectStore";
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct DAClientConfig {
+    pub client: DAClient,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+#[serde(tag = "client")]
+pub enum DAClient {
+    Avail(AvailConfig),
+    ObjectStore(ObjectStoreConfig),
+}

core/lib/config/src/configs/general.rs

@@ -3,6 +3,7 @@ use crate::{
         base_token_adjuster::BaseTokenAdjusterConfig,
         chain::{CircuitBreakerConfig, MempoolConfig, OperationsManagerConfig, StateKeeperConfig},
         consensus::ConsensusConfig,
+        da_client::DAClientConfig,
         da_dispatcher::DADispatcherConfig,
         fri_prover_group::FriProverGroupConfig,
         house_keeper::HouseKeeperConfig,
@@ -41,6 +42,7 @@ pub struct GeneralConfig {
     pub eth: Option<EthConfig>,
     pub snapshot_creator: Option<SnapshotsCreatorConfig>,
     pub observability: Option<ObservabilityConfig>,
+    pub da_client_config: Option<DAClientConfig>,
     pub da_dispatcher_config: Option<DADispatcherConfig>,
     pub protective_reads_writer_config: Option<ProtectiveReadsWriterConfig>,
     pub basic_witness_input_producer_config: Option<BasicWitnessInputProducerConfig>,

core/lib/config/src/configs/mod.rs

@@ -5,6 +5,7 @@ pub use self::{
     commitment_generator::CommitmentGeneratorConfig,
     contract_verifier::ContractVerifierConfig,
     contracts::{ContractsConfig, EcosystemContracts},
+    da_client::{avail::AvailConfig, DAClientConfig},
     da_dispatcher::DADispatcherConfig,
     database::{DBConfig, PostgresConfig},
     eth_sender::{EthConfig, GasAdjusterConfig},
@@ -38,6 +39,7 @@ mod commitment_generator;
 pub mod consensus;
 pub mod contract_verifier;
 pub mod contracts;
+pub mod da_client;
 pub mod da_dispatcher;
 pub mod database;
 pub mod en_config;

core/lib/config/src/lib.rs

@@ -1,9 +1,10 @@
 #![allow(clippy::upper_case_acronyms, clippy::derive_partial_eq_without_eq)]
 
 pub use crate::configs::{
-    ApiConfig, BaseTokenAdjusterConfig, ContractVerifierConfig, ContractsConfig,
-    DADispatcherConfig, DBConfig, EthConfig, EthWatchConfig, ExternalProofIntegrationApiConfig,
-    GasAdjusterConfig, GenesisConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig,
+    ApiConfig, AvailConfig, BaseTokenAdjusterConfig, ContractVerifierConfig, ContractsConfig,
+    DAClientConfig, DADispatcherConfig, DBConfig, EthConfig, EthWatchConfig,
+    ExternalProofIntegrationApiConfig, GasAdjusterConfig, GenesisConfig, ObjectStoreConfig,
+    PostgresConfig, SnapshotsCreatorConfig,
 };
 
 pub mod configs;

core/lib/config/src/testonly.rs

@@ -12,8 +12,12 @@ use zksync_basic_types::{
 use zksync_consensus_utils::EncodeDist;
 use zksync_crypto_primitives::K256PrivateKey;
 
-use crate::configs::{
-    self, eth_sender::PubdataSendingMode, external_price_api_client::ForcedPriceClientConfig,
+use crate::{
+    configs::{
+        self, da_client::DAClient::Avail, eth_sender::PubdataSendingMode,
+        external_price_api_client::ForcedPriceClientConfig,
+    },
+    AvailConfig,
 };
 
 trait Sample {
@@ -922,6 +926,21 @@ impl Distribution<configs::en_config::ENConfig> for EncodeDist {
     }
 }
 
+impl Distribution<configs::da_client::DAClientConfig> for EncodeDist {
+    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::da_client::DAClientConfig {
+        configs::da_client::DAClientConfig {
+            client: Avail(AvailConfig {
+                api_node_url: self.sample(rng),
+                bridge_api_url: self.sample(rng),
+                seed: self.sample(rng),
+                app_id: self.sample(rng),
+                timeout: self.sample(rng),
+                max_retries: self.sample(rng),
+            }),
+        }
+    }
+}
+
 impl Distribution<configs::da_dispatcher::DADispatcherConfig> for EncodeDist {
     fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::da_dispatcher::DADispatcherConfig {
         configs::da_dispatcher::DADispatcherConfig {
@@ -1121,6 +1140,7 @@ impl Distribution<configs::GeneralConfig> for EncodeDist {
             eth: self.sample(rng),
             snapshot_creator: self.sample(rng),
             observability: self.sample(rng),
+            da_client_config: self.sample(rng),
             da_dispatcher_config: self.sample(rng),
             protective_reads_writer_config: self.sample(rng),
             basic_witness_input_producer_config: self.sample(rng),

core/lib/da_client/src/types.rs

@@ -35,6 +35,12 @@ pub struct DispatchResponse {
     pub blob_id: String,
 }
 
+impl From<String> for DispatchResponse {
+    fn from(blob_id: String) -> Self {
+        DispatchResponse { blob_id }
+    }
+}
+
 /// `InclusionData` is the data needed to verify on L1 that a blob is included in the DA layer.
 #[derive(Default, Serialize)]
 pub struct InclusionData {