This repository has been archived by the owner on Nov 15, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
PVF: Refactor workers into separate crates, remove host dependency (#…
…7253) * PVF: Refactor workers into separate crates, remove host dependency * Fix compile error * Remove some leftover code * Fix compile errors * Update Cargo.lock * Remove worker main.rs files I accidentally copied these from the other PR. This PR isn't intended to introduce standalone workers yet. * Address review comments * cargo fmt * Update a couple of comments * Update log targets
- Loading branch information
Showing
50 changed files
with
773 additions
and
515 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| [package] | ||
| name = "polkadot-node-core-pvf-common" | ||
| version.workspace = true | ||
| authors.workspace = true | ||
| edition.workspace = true | ||
|
|
||
| [dependencies] | ||
| cpu-time = "1.0.0" | ||
| futures = "0.3.21" | ||
| gum = { package = "tracing-gum", path = "../../../gum" } | ||
| libc = "0.2.139" | ||
| tokio = { version = "1.24.2", features = ["fs", "process", "io-util"] } | ||
|
|
||
| parity-scale-codec = { version = "3.4.0", default-features = false, features = ["derive"] } | ||
|
|
||
| polkadot-parachain = { path = "../../../../parachain" } | ||
| polkadot-primitives = { path = "../../../../primitives" } | ||
|
|
||
| sc-executor-common = { git = "https://github.com/paritytech/substrate", branch = "master" } | ||
| sc-executor-wasmtime = { git = "https://github.com/paritytech/substrate", branch = "master" } | ||
|
|
||
| sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } | ||
| sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } | ||
|
|
||
| [build-dependencies] | ||
| substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| // Copyright (C) Parity Technologies (UK) Ltd. | ||
| // This file is part of Polkadot. | ||
|
|
||
| // Polkadot is free software: you can redistribute it and/or modify | ||
| // it under the terms of the GNU General Public License as published by | ||
| // the Free Software Foundation, either version 3 of the License, or | ||
| // (at your option) any later version. | ||
|
|
||
| // Polkadot is distributed in the hope that it will be useful, | ||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| // GNU General Public License for more details. | ||
|
|
||
| // You should have received a copy of the GNU General Public License | ||
| // along with Polkadot. If not, see <http://www.gnu.org/licenses/>. | ||
|
|
||
| use crate::prepare::PrepareStats; | ||
| use parity_scale_codec::{Decode, Encode}; | ||
| use std::fmt; | ||
|
|
||
| /// Result of PVF preparation performed by the validation host. Contains stats about the preparation if | ||
| /// successful | ||
| pub type PrepareResult = Result<PrepareStats, PrepareError>; | ||
|
|
||
| /// An error that occurred during the prepare part of the PVF pipeline. | ||
| #[derive(Debug, Clone, Encode, Decode)] | ||
| pub enum PrepareError { | ||
| /// During the prevalidation stage of preparation an issue was found with the PVF. | ||
| Prevalidation(String), | ||
| /// Compilation failed for the given PVF. | ||
| Preparation(String), | ||
| /// An unexpected panic has occurred in the preparation worker. | ||
| Panic(String), | ||
| /// Failed to prepare the PVF due to the time limit. | ||
| TimedOut, | ||
| /// An IO error occurred. This state is reported by either the validation host or by the worker. | ||
| IoErr(String), | ||
| /// The temporary file for the artifact could not be created at the given cache path. This state is reported by the | ||
| /// validation host (not by the worker). | ||
| CreateTmpFileErr(String), | ||
| /// The response from the worker is received, but the file cannot be renamed (moved) to the final destination | ||
| /// location. This state is reported by the validation host (not by the worker). | ||
| RenameTmpFileErr(String), | ||
| } | ||
|
|
||
| impl PrepareError { | ||
| /// Returns whether this is a deterministic error, i.e. one that should trigger reliably. Those | ||
| /// errors depend on the PVF itself and the sc-executor/wasmtime logic. | ||
| /// | ||
| /// Non-deterministic errors can happen spuriously. Typically, they occur due to resource | ||
| /// starvation, e.g. under heavy load or memory pressure. Those errors are typically transient | ||
| /// but may persist e.g. if the node is run by overwhelmingly underpowered machine. | ||
| pub fn is_deterministic(&self) -> bool { | ||
| use PrepareError::*; | ||
| match self { | ||
| Prevalidation(_) | Preparation(_) | Panic(_) => true, | ||
| TimedOut | IoErr(_) | CreateTmpFileErr(_) | RenameTmpFileErr(_) => false, | ||
| } | ||
| } | ||
| } | ||
|
|
||
| impl fmt::Display for PrepareError { | ||
| fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | ||
| use PrepareError::*; | ||
| match self { | ||
| Prevalidation(err) => write!(f, "prevalidation: {}", err), | ||
| Preparation(err) => write!(f, "preparation: {}", err), | ||
| Panic(err) => write!(f, "panic: {}", err), | ||
| TimedOut => write!(f, "prepare: timeout"), | ||
| IoErr(err) => write!(f, "prepare: io error while receiving response: {}", err), | ||
| CreateTmpFileErr(err) => write!(f, "prepare: error creating tmp file: {}", err), | ||
| RenameTmpFileErr(err) => write!(f, "prepare: error renaming tmp file: {}", err), | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /// Some internal error occurred. | ||
| /// | ||
| /// Should only ever be used for validation errors independent of the candidate and PVF, or for errors we ruled out | ||
| /// during pre-checking (so preparation errors are fine). | ||
| #[derive(Debug, Clone, Encode, Decode)] | ||
| pub enum InternalValidationError { | ||
| /// Some communication error occurred with the host. | ||
| HostCommunication(String), | ||
| /// Could not find or open compiled artifact file. | ||
| CouldNotOpenFile(String), | ||
| /// An error occurred in the CPU time monitor thread. Should be totally unrelated to validation. | ||
| CpuTimeMonitorThread(String), | ||
| /// Some non-deterministic preparation error occurred. | ||
| NonDeterministicPrepareError(PrepareError), | ||
| } | ||
|
|
||
| impl fmt::Display for InternalValidationError { | ||
| fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | ||
| use InternalValidationError::*; | ||
| match self { | ||
| HostCommunication(err) => | ||
| write!(f, "validation: some communication error occurred with the host: {}", err), | ||
| CouldNotOpenFile(err) => | ||
| write!(f, "validation: could not find or open compiled artifact file: {}", err), | ||
| CpuTimeMonitorThread(err) => | ||
| write!(f, "validation: an error occurred in the CPU time monitor thread: {}", err), | ||
| NonDeterministicPrepareError(err) => write!(f, "validation: prepare: {}", err), | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.