From f24c3eb6684b5fab2c3ed74312e1ea797304ab95 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Wed, 29 May 2024 16:00:41 -0700 Subject: [PATCH 1/9] feat(json_schema): Add support for JSON schema generation --- Cargo.lock | 37 ++++++++++++++++ Cargo.toml | 1 + crates/weaver_resolved_schema/Cargo.toml | 1 + .../weaver_resolved_schema/src/attribute.rs | 5 ++- crates/weaver_resolved_schema/src/catalog.rs | 3 +- crates/weaver_resolved_schema/src/lib.rs | 15 ++++++- crates/weaver_resolved_schema/src/lineage.rs | 5 ++- crates/weaver_resolved_schema/src/registry.rs | 7 ++-- crates/weaver_resolved_schema/src/tags.rs | 3 +- crates/weaver_resolved_schema/src/value.rs | 3 +- crates/weaver_semconv/Cargo.toml | 1 + crates/weaver_semconv/src/attribute.rs | 42 +++++++++++++++---- crates/weaver_semconv/src/group.rs | 7 ++-- crates/weaver_semconv/src/stability.rs | 3 +- 14 files changed, 111 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d6fe01c1..0d8e8616 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2940,6 +2940,30 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schemars" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1eee588578aff73f856ab961cd2f79e36bc45d7ded33a7562adba4667aecc0e" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + [[package]] name = "scientific" version = "0.5.2" @@ -3001,6 +3025,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_json" version = "1.0.117" @@ -3839,6 +3874,7 @@ name = "weaver_resolved_schema" version = "0.1.0" dependencies = [ "ordered-float", + "schemars", "serde", "serde_json", "thiserror", @@ -3871,6 +3907,7 @@ dependencies = [ "glob", "miette", "ordered-float", + "schemars", "serde", "serde_yaml", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index 7e7e51b0..c1f78589 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ globset = { version = "0.4.14", features = ["serde1"] } miette = { version = "7.2.0", features = ["fancy", "serde"] } include_dir = "0.7.3" tempdir = "0.3.7" +schemars = "0.8.21" # Features definition ========================================================= [features] diff --git a/crates/weaver_resolved_schema/Cargo.toml b/crates/weaver_resolved_schema/Cargo.toml index 6ab55312..0bcafde9 100644 --- a/crates/weaver_resolved_schema/Cargo.toml +++ b/crates/weaver_resolved_schema/Cargo.toml @@ -18,6 +18,7 @@ weaver_semconv = { path = "../weaver_semconv" } thiserror.workspace = true serde.workspace = true ordered-float.workspace = true +schemars.workspace = true [dev-dependencies] serde_json.workspace = true diff --git a/crates/weaver_resolved_schema/src/attribute.rs b/crates/weaver_resolved_schema/src/attribute.rs index 8a72acef..b39df2ab 100644 --- a/crates/weaver_resolved_schema/src/attribute.rs +++ b/crates/weaver_resolved_schema/src/attribute.rs @@ -8,11 +8,12 @@ use crate::tags::Tags; use crate::value::Value; use serde::{Deserialize, Serialize}; use std::fmt::Display; +use schemars::JsonSchema; use weaver_semconv::attribute::{AttributeSpec, AttributeType, Examples, RequirementLevel}; use weaver_semconv::stability::Stability; /// An attribute definition. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Attribute { /// Attribute name. @@ -81,7 +82,7 @@ pub struct UnresolvedAttribute { } /// An internal reference to an attribute in the catalog. -#[derive(Serialize, Deserialize, Debug, Copy, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[derive(Serialize, Deserialize, Debug, Copy, Clone, Eq, PartialEq, PartialOrd, Ord, JsonSchema)] pub struct AttributeRef(pub u32); impl Display for AttributeRef { diff --git a/crates/weaver_resolved_schema/src/catalog.rs b/crates/weaver_resolved_schema/src/catalog.rs index 311e26c9..7c3d5f2c 100644 --- a/crates/weaver_resolved_schema/src/catalog.rs +++ b/crates/weaver_resolved_schema/src/catalog.rs @@ -7,6 +7,7 @@ use crate::attribute::{Attribute, AttributeRef}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; use std::fmt::Debug; +use schemars::JsonSchema; use weaver_semconv::attribute::{AttributeType, BasicRequirementLevelSpec, RequirementLevel}; use weaver_semconv::stability::Stability; @@ -14,7 +15,7 @@ use weaver_semconv::stability::Stability; /// Attribute references are used to refer to attributes in the catalog. /// /// Note : In the future, this catalog could be extended with other entities. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] #[serde(deny_unknown_fields)] #[must_use] pub struct Catalog { diff --git a/crates/weaver_resolved_schema/src/lib.rs b/crates/weaver_resolved_schema/src/lib.rs index 724d0a52..0ae8166b 100644 --- a/crates/weaver_resolved_schema/src/lib.rs +++ b/crates/weaver_resolved_schema/src/lib.rs @@ -10,6 +10,7 @@ use crate::registry::Registry; use crate::resource::Resource; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use schemars::JsonSchema; use weaver_version::Versions; pub mod attribute; @@ -31,7 +32,7 @@ pub const OTEL_REGISTRY_ID: &str = "OTEL"; /// A Resolved Telemetry Schema. /// A Resolved Telemetry Schema is self-contained and doesn't contain any /// external references to other schemas or semantic conventions. -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, JsonSchema)] #[serde(deny_unknown_fields)] pub struct ResolvedTelemetrySchema { /// Version of the file structure. @@ -102,3 +103,15 @@ impl ResolvedTelemetrySchema { } } } + +#[cfg(test)] +mod tests { + use schemars::schema_for; + use crate::ResolvedTelemetrySchema; + + #[test] + fn test_json_schema() { + let schema = schema_for!(ResolvedTelemetrySchema); + println!("{}", serde_json::to_string_pretty(&schema).unwrap()); + } +} \ No newline at end of file diff --git a/crates/weaver_resolved_schema/src/lineage.rs b/crates/weaver_resolved_schema/src/lineage.rs index 5c8cdd0b..4c2bda39 100644 --- a/crates/weaver_resolved_schema/src/lineage.rs +++ b/crates/weaver_resolved_schema/src/lineage.rs @@ -3,6 +3,7 @@ //! Data structures used to keep track of the lineage of a semantic convention. use std::collections::{BTreeMap, BTreeSet}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -10,7 +11,7 @@ use weaver_semconv::attribute::{AttributeSpec, Examples, RequirementLevel}; use weaver_semconv::stability::Stability; /// Attribute lineage (at the field level). -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] #[serde(rename_all = "snake_case")] pub struct AttributeLineage { /// The group id where the attribute is coming from. @@ -28,7 +29,7 @@ pub struct AttributeLineage { } /// Group lineage. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] #[must_use] pub struct GroupLineage { /// The path or URL of the source file where the group is defined. diff --git a/crates/weaver_resolved_schema/src/registry.rs b/crates/weaver_resolved_schema/src/registry.rs index 27fe64f2..ad067d03 100644 --- a/crates/weaver_resolved_schema/src/registry.rs +++ b/crates/weaver_resolved_schema/src/registry.rs @@ -5,6 +5,7 @@ //! A semantic convention registry. use std::collections::{BTreeMap, HashMap, HashSet}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -20,7 +21,7 @@ use crate::registry::GroupStats::{ }; /// A semantic convention registry. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Registry { /// The semantic convention registry url. @@ -43,7 +44,7 @@ pub struct Stats { } /// Group specification. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] pub struct Group { /// The id that uniquely identifies the semantic convention. pub id: String, @@ -193,7 +194,7 @@ pub enum GroupStats { } /// Allow to define additional requirements on the semantic convention. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Hash, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Hash, Eq, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Constraint { /// any_of accepts a list of sequences. Each sequence contains a list of diff --git a/crates/weaver_resolved_schema/src/tags.rs b/crates/weaver_resolved_schema/src/tags.rs index 572dfe86..e1e357a6 100644 --- a/crates/weaver_resolved_schema/src/tags.rs +++ b/crates/weaver_resolved_schema/src/tags.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; +use schemars::JsonSchema; /// A set of tags. /// @@ -15,7 +16,7 @@ use std::collections::BTreeMap; /// - semantic_type: first_name /// - owner: /// - provenance: browser_sensor -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Hash)] +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Hash, JsonSchema)] #[serde(transparent)] #[serde(deny_unknown_fields)] pub struct Tags { diff --git a/crates/weaver_resolved_schema/src/value.rs b/crates/weaver_resolved_schema/src/value.rs index 60cb9f99..41a8c370 100644 --- a/crates/weaver_resolved_schema/src/value.rs +++ b/crates/weaver_resolved_schema/src/value.rs @@ -3,10 +3,11 @@ //! Specification of a resolved value. use ordered_float::OrderedFloat; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// The different types of values. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] #[serde(tag = "type")] #[must_use] pub enum Value { diff --git a/crates/weaver_semconv/Cargo.toml b/crates/weaver_semconv/Cargo.toml index 70c0c7c0..97c2fc12 100644 --- a/crates/weaver_semconv/Cargo.toml +++ b/crates/weaver_semconv/Cargo.toml @@ -20,5 +20,6 @@ thiserror.workspace = true ureq.workspace = true ordered-float.workspace = true miette.workspace = true +schemars.workspace = true glob = "0.3.1" \ No newline at end of file diff --git a/crates/weaver_semconv/src/attribute.rs b/crates/weaver_semconv/src/attribute.rs index c6fd215d..1fa059f1 100644 --- a/crates/weaver_semconv/src/attribute.rs +++ b/crates/weaver_semconv/src/attribute.rs @@ -4,9 +4,13 @@ //! Attribute specification. +use std::borrow::Cow; use ordered_float::OrderedFloat; use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; +use schemars::gen::SchemaGenerator; +use schemars::JsonSchema; +use schemars::schema::{InstanceType, Schema, SchemaObject}; use crate::stability::Stability; @@ -175,7 +179,7 @@ impl AttributeSpec { } /// The different types of attributes (specification). -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] #[serde(rename_all = "snake_case")] #[serde(untagged)] pub enum AttributeType { @@ -218,7 +222,7 @@ fn default_as_true() -> bool { } /// Primitive or array types. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum PrimitiveOrArrayTypeSpec { /// A boolean attribute. @@ -260,7 +264,7 @@ impl Display for PrimitiveOrArrayTypeSpec { } /// Template types. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum TemplateTypeSpec { /// A boolean attribute. @@ -306,7 +310,7 @@ impl Display for TemplateTypeSpec { } /// Possible enum entries. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] #[serde(deny_unknown_fields)] pub struct EnumEntriesSpec { /// String that uniquely identifies the enum entry. @@ -345,6 +349,30 @@ pub enum ValueSpec { String(String), } +// ToDo +// impl JsonSchema for OrderedFloat { +// fn is_referenceable() -> bool { +// false +// } +// +// fn schema_name() -> String { +// "double".to_owned() +// } +// +// fn schema_id() -> Cow<'static, str> { +// Cow::Borrowed("double") +// } +// +// fn json_schema(_: &mut SchemaGenerator) -> Schema { +// SchemaObject { +// instance_type: Some(InstanceType::Number.into()), +// format: (Some("double".to_owned())), +// ..Default::default() +// } +// .into() +// } +// } + /// Implements a human readable display for Value. impl Display for ValueSpec { /// Formats the value. @@ -390,7 +418,7 @@ impl From<&str> for ValueSpec { } /// The different types of examples. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] #[serde(rename_all = "snake_case")] #[serde(untagged)] pub enum Examples { @@ -421,7 +449,7 @@ pub enum Examples { } /// The different requirement level specifications. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] #[serde(rename_all = "snake_case")] #[serde(untagged)] pub enum RequirementLevel { @@ -463,7 +491,7 @@ impl Default for RequirementLevel { } /// The different types of basic requirement levels. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum BasicRequirementLevelSpec { /// A required requirement level. diff --git a/crates/weaver_semconv/src/group.rs b/crates/weaver_semconv/src/group.rs index 3dacc4e7..9a9b6af6 100644 --- a/crates/weaver_semconv/src/group.rs +++ b/crates/weaver_semconv/src/group.rs @@ -5,6 +5,7 @@ //! A group specification. use std::fmt::{Display, Formatter}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -204,7 +205,7 @@ impl GroupSpec { } /// The different types of groups (specification). -#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum GroupType { /// Attribute group (attribute_group type) defines a set of attributes that @@ -236,7 +237,7 @@ impl Default for GroupType { } /// The span kind. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum SpanKindSpec { /// An internal span. @@ -267,7 +268,7 @@ pub struct ConstraintSpec { } /// The type of the metric. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum InstrumentSpec { /// An up-down counter metric. diff --git a/crates/weaver_semconv/src/stability.rs b/crates/weaver_semconv/src/stability.rs index 644b0ac0..e8dd86c1 100644 --- a/crates/weaver_semconv/src/stability.rs +++ b/crates/weaver_semconv/src/stability.rs @@ -4,9 +4,10 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; +use schemars::JsonSchema; /// The level of stability for a definition. -#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Hash, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum Stability { /// A deprecated definition. From ed5f7fc27496d6c1ab9549cd2c2d54ee328e0d76 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Wed, 29 May 2024 16:52:35 -0700 Subject: [PATCH 2/9] feat(json_schema): Add support for JSON schema generation for both ResolvedRegistry and ResolvedTelemetrySchema TemplateRegistry had been renamed ResolvedRegistry. --- Cargo.lock | 3 + Cargo.toml | 2 +- crates/weaver_codegen_test/build.rs | 4 +- crates/weaver_forge/Cargo.toml | 1 + .../weaver_forge/allowed-external-types.toml | 1 + crates/weaver_forge/src/lib.rs | 12 +- crates/weaver_forge/src/registry.rs | 40 ++++-- .../allowed-external-types.toml | 1 + .../weaver_resolved_schema/src/attribute.rs | 6 +- crates/weaver_resolved_schema/src/catalog.rs | 2 +- .../src/instrumentation_library.rs | 3 +- crates/weaver_resolved_schema/src/lib.rs | 14 +- crates/weaver_resolved_schema/src/lineage.rs | 2 +- crates/weaver_resolved_schema/src/metric.rs | 7 +- crates/weaver_resolved_schema/src/registry.rs | 2 +- crates/weaver_resolved_schema/src/resource.rs | 3 +- crates/weaver_resolved_schema/src/signal.rs | 15 +- crates/weaver_resolved_schema/src/tags.rs | 2 +- .../allowed-external-types.toml | 1 + crates/weaver_semconv/src/attribute.rs | 31 +---- crates/weaver_semconv/src/group.rs | 2 +- crates/weaver_semconv/src/stability.rs | 2 +- crates/weaver_semconv_gen/src/lib.rs | 6 +- crates/weaver_version/Cargo.toml | 1 + .../allowed-external-types.toml | 1 + crates/weaver_version/src/lib.rs | 5 +- crates/weaver_version/src/logs_change.rs | 5 +- crates/weaver_version/src/logs_version.rs | 3 +- crates/weaver_version/src/metrics_change.rs | 5 +- crates/weaver_version/src/metrics_version.rs | 3 +- crates/weaver_version/src/resource_change.rs | 5 +- crates/weaver_version/src/resource_version.rs | 3 +- crates/weaver_version/src/spans_change.rs | 5 +- crates/weaver_version/src/spans_version.rs | 3 +- docs/images/dependencies.svg | 130 +++++++++--------- src/registry/generate.rs | 4 +- src/registry/resolve.rs | 4 +- 37 files changed, 182 insertions(+), 157 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d8e8616..ade2fdb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2407,6 +2407,7 @@ checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" dependencies = [ "num-traits", "rand 0.8.5", + "schemars", "serde", ] @@ -3857,6 +3858,7 @@ dependencies = [ "opentelemetry_sdk", "rayon", "regex", + "schemars", "serde", "serde_json", "serde_yaml", @@ -3937,6 +3939,7 @@ dependencies = [ name = "weaver_version" version = "0.1.0" dependencies = [ + "schemars", "semver", "serde", "serde_yaml", diff --git a/Cargo.toml b/Cargo.toml index c1f78589..676213ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ thiserror = "1.0.58" ureq = "2.9.7" regex = "1.10.3" rayon = "1.10.0" -ordered-float = { version = "4.2.0", features = ["serde"] } +ordered-float = { version = "4.2.0", features = ["serde", "schemars"] } walkdir = "2.5.0" anyhow = "1.0.83" itertools = "0.12.1" diff --git a/crates/weaver_codegen_test/build.rs b/crates/weaver_codegen_test/build.rs index e1974fc2..8ef16fb2 100644 --- a/crates/weaver_codegen_test/build.rs +++ b/crates/weaver_codegen_test/build.rs @@ -14,7 +14,7 @@ use weaver_cache::Cache; use weaver_common::in_memory::LogMessage; use weaver_common::{in_memory, Logger}; use weaver_forge::file_loader::FileSystemFileLoader; -use weaver_forge::registry::TemplateRegistry; +use weaver_forge::registry::ResolvedRegistry; use weaver_forge::{OutputDirective, TemplateEngine}; use weaver_resolver::SchemaResolver; use weaver_semconv::path::RegistryPath; @@ -51,7 +51,7 @@ fn main() { let loader = FileSystemFileLoader::try_new(TEMPLATES_PATH.into(), TARGET) .unwrap_or_else(|e| process_error(&logger, e)); let engine = TemplateEngine::try_new(loader).unwrap_or_else(|e| process_error(&logger, e)); - let template_registry = TemplateRegistry::try_from_resolved_registry( + let template_registry = ResolvedRegistry::try_from_resolved_registry( schema .registry(REGISTRY_ID) .expect("Failed to get the registry from the resolved schema"), diff --git a/crates/weaver_forge/Cargo.toml b/crates/weaver_forge/Cargo.toml index 7d04b8d9..c06dbb99 100644 --- a/crates/weaver_forge/Cargo.toml +++ b/crates/weaver_forge/Cargo.toml @@ -38,6 +38,7 @@ walkdir.workspace = true globset.workspace = true miette.workspace = true include_dir.workspace = true +schemars.workspace = true [dev-dependencies] opentelemetry = { version = "0.22.0", features = ["trace", "metrics", "logs", "otel_unstable"] } diff --git a/crates/weaver_forge/allowed-external-types.toml b/crates/weaver_forge/allowed-external-types.toml index 4994cbd8..09302ea7 100644 --- a/crates/weaver_forge/allowed-external-types.toml +++ b/crates/weaver_forge/allowed-external-types.toml @@ -12,4 +12,5 @@ allowed_external_types = [ "minijinja::value::Value", "miette::protocol::Diagnostic", "include_dir::dir::Dir", + "schemars::JsonSchema", ] \ No newline at end of file diff --git a/crates/weaver_forge/src/lib.rs b/crates/weaver_forge/src/lib.rs index 605c1b13..3197fff3 100644 --- a/crates/weaver_forge/src/lib.rs +++ b/crates/weaver_forge/src/lib.rs @@ -29,7 +29,7 @@ use crate::debug::error_summary; use crate::error::Error::InvalidConfigFile; use crate::extensions::{ansi, case, code, otel, util}; use crate::file_loader::FileLoader; -use crate::registry::{TemplateGroup, TemplateRegistry}; +use crate::registry::{ResolvedGroup, ResolvedRegistry}; mod config; pub mod debug; @@ -109,11 +109,11 @@ pub struct TemplateEngine { #[derive(Serialize, Debug)] pub struct Context<'a> { /// The semantic convention registry. - pub registry: &'a TemplateRegistry, + pub registry: &'a ResolvedRegistry, /// The group to generate doc or code for. - pub group: Option<&'a TemplateGroup>, + pub group: Option<&'a ResolvedGroup>, /// The groups to generate doc or code for. - pub groups: Option>, + pub groups: Option>, } /// Global context for the template engine. @@ -432,7 +432,7 @@ mod tests { use crate::extensions::case::case_converter; use crate::file_loader::FileSystemFileLoader; use crate::filter::Filter; - use crate::registry::TemplateRegistry; + use crate::registry::ResolvedRegistry; use crate::OutputDirective; #[test] @@ -580,7 +580,7 @@ mod tests { let schema = SchemaResolver::resolve_semantic_convention_registry(&mut registry) .expect("Failed to resolve registry"); - let template_registry = TemplateRegistry::try_from_resolved_registry( + let template_registry = ResolvedRegistry::try_from_resolved_registry( schema.registry(registry_id).expect("registry not found"), schema.catalog(), ) diff --git a/crates/weaver_forge/src/registry.rs b/crates/weaver_forge/src/registry.rs index 1b851a12..9d307af4 100644 --- a/crates/weaver_forge/src/registry.rs +++ b/crates/weaver_forge/src/registry.rs @@ -5,6 +5,7 @@ //! evaluation. use crate::error::Error; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use weaver_resolved_schema::attribute::Attribute; use weaver_resolved_schema::catalog::Catalog; @@ -13,20 +14,21 @@ use weaver_resolved_schema::registry::{Constraint, Group, Registry}; use weaver_semconv::group::{GroupType, InstrumentSpec, SpanKindSpec}; use weaver_semconv::stability::Stability; -/// A semantic convention registry used in the context of the template engine. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +/// A resolved semantic convention registry used in the context of the template and policy +/// engines. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] #[serde(deny_unknown_fields)] -pub struct TemplateRegistry { +pub struct ResolvedRegistry { /// The semantic convention registry url. #[serde(skip_serializing_if = "String::is_empty")] pub registry_url: String, /// A list of semantic convention groups. - pub groups: Vec, + pub groups: Vec, } -/// Group specification used in the context of the template engine. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct TemplateGroup { +/// Resolved group specification used in the context of the template engine. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +pub struct ResolvedGroup { /// The id that uniquely identifies the semantic convention. pub id: String, /// The type of the group including the specific fields for each type. @@ -102,7 +104,7 @@ pub struct TemplateGroup { pub lineage: Option, } -impl TemplateGroup { +impl ResolvedGroup { /// Constructs a Template-friendly groups structure from resolved registry structures. pub fn try_from_resolved(group: &Group, catalog: &Catalog) -> Result { let mut errors = Vec::new(); @@ -133,7 +135,7 @@ impl TemplateGroup { if !errors.is_empty() { return Err(Error::CompoundError(errors)); } - Ok(TemplateGroup { + Ok(ResolvedGroup { id, r#type: group_type, brief, @@ -155,7 +157,7 @@ impl TemplateGroup { } } -impl TemplateRegistry { +impl ResolvedRegistry { /// Create a new template registry from a resolved registry. pub fn try_from_resolved_registry( registry: &Registry, @@ -191,7 +193,7 @@ impl TemplateRegistry { }) .collect(); let lineage = group.lineage.clone(); - TemplateGroup { + ResolvedGroup { id, r#type: group_type, brief, @@ -223,3 +225,19 @@ impl TemplateRegistry { }) } } + +#[cfg(test)] +mod tests { + use crate::ResolvedRegistry; + use schemars::schema_for; + use serde_json::to_string_pretty; + + #[test] + fn test_json_schema_gen() { + // Ensure the JSON schema can be generated for the TemplateRegistry + let schema = schema_for!(ResolvedRegistry); + + // Ensure the schema can be serialized to a string + assert!(to_string_pretty(&schema).is_ok()); + } +} diff --git a/crates/weaver_resolved_schema/allowed-external-types.toml b/crates/weaver_resolved_schema/allowed-external-types.toml index c290511a..ee279afb 100644 --- a/crates/weaver_resolved_schema/allowed-external-types.toml +++ b/crates/weaver_resolved_schema/allowed-external-types.toml @@ -8,4 +8,5 @@ allowed_external_types = [ "ordered_float::OrderedFloat", # ToDo: Remove this dependency before version 1.0 "weaver_semconv::*", "weaver_version::*", + "schemars::JsonSchema", ] \ No newline at end of file diff --git a/crates/weaver_resolved_schema/src/attribute.rs b/crates/weaver_resolved_schema/src/attribute.rs index b39df2ab..09acd37f 100644 --- a/crates/weaver_resolved_schema/src/attribute.rs +++ b/crates/weaver_resolved_schema/src/attribute.rs @@ -6,9 +6,9 @@ use crate::tags::Tags; use crate::value::Value; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::fmt::Display; -use schemars::JsonSchema; use weaver_semconv::attribute::{AttributeSpec, AttributeType, Examples, RequirementLevel}; use weaver_semconv::stability::Stability; @@ -82,7 +82,9 @@ pub struct UnresolvedAttribute { } /// An internal reference to an attribute in the catalog. -#[derive(Serialize, Deserialize, Debug, Copy, Clone, Eq, PartialEq, PartialOrd, Ord, JsonSchema)] +#[derive( + Serialize, Deserialize, Debug, Copy, Clone, Eq, PartialEq, PartialOrd, Ord, JsonSchema, +)] pub struct AttributeRef(pub u32); impl Display for AttributeRef { diff --git a/crates/weaver_resolved_schema/src/catalog.rs b/crates/weaver_resolved_schema/src/catalog.rs index 7c3d5f2c..8fe1046d 100644 --- a/crates/weaver_resolved_schema/src/catalog.rs +++ b/crates/weaver_resolved_schema/src/catalog.rs @@ -4,10 +4,10 @@ //! that are shared across multiple signals in the Resolved Telemetry Schema. use crate::attribute::{Attribute, AttributeRef}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; use std::fmt::Debug; -use schemars::JsonSchema; use weaver_semconv::attribute::{AttributeType, BasicRequirementLevelSpec, RequirementLevel}; use weaver_semconv::stability::Stability; diff --git a/crates/weaver_resolved_schema/src/instrumentation_library.rs b/crates/weaver_resolved_schema/src/instrumentation_library.rs index 907ef3d6..3ba47f4f 100644 --- a/crates/weaver_resolved_schema/src/instrumentation_library.rs +++ b/crates/weaver_resolved_schema/src/instrumentation_library.rs @@ -4,11 +4,12 @@ use crate::signal::{Event, MultivariateMetric, Span, UnivariateMetric}; use crate::tags::Tags; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// An instrumentation library specification. /// MUST be used both by applications and libraries. -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, JsonSchema)] #[serde(deny_unknown_fields)] pub struct InstrumentationLibrary { /// An optional name for the instrumentation library. diff --git a/crates/weaver_resolved_schema/src/lib.rs b/crates/weaver_resolved_schema/src/lib.rs index 0ae8166b..61ed0ed7 100644 --- a/crates/weaver_resolved_schema/src/lib.rs +++ b/crates/weaver_resolved_schema/src/lib.rs @@ -8,9 +8,9 @@ use crate::catalog::Catalog; use crate::instrumentation_library::InstrumentationLibrary; use crate::registry::Registry; use crate::resource::Resource; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use schemars::JsonSchema; use weaver_version::Versions; pub mod attribute; @@ -106,12 +106,16 @@ impl ResolvedTelemetrySchema { #[cfg(test)] mod tests { - use schemars::schema_for; use crate::ResolvedTelemetrySchema; + use schemars::schema_for; + use serde_json::to_string_pretty; #[test] - fn test_json_schema() { + fn test_json_schema_gen() { + // Ensure the JSON schema can be generated for the ResolvedTelemetrySchema let schema = schema_for!(ResolvedTelemetrySchema); - println!("{}", serde_json::to_string_pretty(&schema).unwrap()); + + // Ensure the schema can be serialized to a string + assert!(to_string_pretty(&schema).is_ok()); } -} \ No newline at end of file +} diff --git a/crates/weaver_resolved_schema/src/lineage.rs b/crates/weaver_resolved_schema/src/lineage.rs index 4c2bda39..9dca09c5 100644 --- a/crates/weaver_resolved_schema/src/lineage.rs +++ b/crates/weaver_resolved_schema/src/lineage.rs @@ -2,8 +2,8 @@ //! Data structures used to keep track of the lineage of a semantic convention. -use std::collections::{BTreeMap, BTreeSet}; use schemars::JsonSchema; +use std::collections::{BTreeMap, BTreeSet}; use serde::{Deserialize, Serialize}; diff --git a/crates/weaver_resolved_schema/src/metric.rs b/crates/weaver_resolved_schema/src/metric.rs index 9a5fd7ce..b0122b1e 100644 --- a/crates/weaver_resolved_schema/src/metric.rs +++ b/crates/weaver_resolved_schema/src/metric.rs @@ -3,14 +3,15 @@ //! Specification of a resolved metric. use crate::tags::Tags; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// An internal reference to a metric in the catalog. -#[derive(Serialize, Deserialize, Debug, Copy, Clone)] +#[derive(Serialize, Deserialize, Debug, Copy, Clone, JsonSchema)] pub struct MetricRef(pub u32); /// A metric definition. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Metric { /// Metric name. @@ -29,7 +30,7 @@ pub struct Metric { } /// The type of the metric. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] pub enum Instrument { /// An up-down counter metric. UpDownCounter, diff --git a/crates/weaver_resolved_schema/src/registry.rs b/crates/weaver_resolved_schema/src/registry.rs index ad067d03..23c2fa7d 100644 --- a/crates/weaver_resolved_schema/src/registry.rs +++ b/crates/weaver_resolved_schema/src/registry.rs @@ -4,8 +4,8 @@ //! A semantic convention registry. -use std::collections::{BTreeMap, HashMap, HashSet}; use schemars::JsonSchema; +use std::collections::{BTreeMap, HashMap, HashSet}; use serde::{Deserialize, Serialize}; diff --git a/crates/weaver_resolved_schema/src/resource.rs b/crates/weaver_resolved_schema/src/resource.rs index e5303f2a..6fff9930 100644 --- a/crates/weaver_resolved_schema/src/resource.rs +++ b/crates/weaver_resolved_schema/src/resource.rs @@ -3,10 +3,11 @@ //! Define an OpenTelemetry resource. use crate::attribute::AttributeRef; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// Definition of attributes associated with the resource. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Resource { /// List of references to attributes present in the shared catalog. diff --git a/crates/weaver_resolved_schema/src/signal.rs b/crates/weaver_resolved_schema/src/signal.rs index b0b60a19..a702191c 100644 --- a/crates/weaver_resolved_schema/src/signal.rs +++ b/crates/weaver_resolved_schema/src/signal.rs @@ -2,6 +2,7 @@ //! Define the concept of signal. +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::attribute::AttributeRef; @@ -9,7 +10,7 @@ use crate::metric::MetricRef; use crate::tags::Tags; /// A univariate metric signal. -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, JsonSchema)] #[serde(deny_unknown_fields)] pub struct UnivariateMetric { /// References to attributes defined in the catalog. @@ -23,7 +24,7 @@ pub struct UnivariateMetric { } /// A multivariate metric signal. -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, JsonSchema)] #[serde(deny_unknown_fields)] pub struct MultivariateMetric { /// The name of the multivariate metric. @@ -44,7 +45,7 @@ pub struct MultivariateMetric { } /// An event signal. -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Event { /// The name of the event. @@ -65,7 +66,7 @@ pub struct Event { } /// A span signal. -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Span { /// The name of the span. @@ -93,7 +94,7 @@ pub struct Span { } /// The span kind. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] pub enum SpanKind { /// An internal span. Internal, @@ -108,7 +109,7 @@ pub enum SpanKind { } /// A span event specification. -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, JsonSchema)] #[serde(deny_unknown_fields)] pub struct SpanEvent { /// The name of the span event. @@ -127,7 +128,7 @@ pub struct SpanEvent { } /// A span link specification. -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, JsonSchema)] #[serde(deny_unknown_fields)] pub struct SpanLink { /// The name of the span link. diff --git a/crates/weaver_resolved_schema/src/tags.rs b/crates/weaver_resolved_schema/src/tags.rs index e1e357a6..a3ff073f 100644 --- a/crates/weaver_resolved_schema/src/tags.rs +++ b/crates/weaver_resolved_schema/src/tags.rs @@ -2,9 +2,9 @@ //! Define the concept of tag. +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -use schemars::JsonSchema; /// A set of tags. /// diff --git a/crates/weaver_semconv/allowed-external-types.toml b/crates/weaver_semconv/allowed-external-types.toml index 5563d5bb..1dbc3fa2 100644 --- a/crates/weaver_semconv/allowed-external-types.toml +++ b/crates/weaver_semconv/allowed-external-types.toml @@ -8,4 +8,5 @@ allowed_external_types = [ "weaver_common::error::WeaverError", "ordered_float::OrderedFloat", # ToDo: Remove this dependency before version 1.0 "miette::protocol::Diagnostic", + "schemars::JsonSchema", ] \ No newline at end of file diff --git a/crates/weaver_semconv/src/attribute.rs b/crates/weaver_semconv/src/attribute.rs index 1fa059f1..019532e7 100644 --- a/crates/weaver_semconv/src/attribute.rs +++ b/crates/weaver_semconv/src/attribute.rs @@ -4,13 +4,10 @@ //! Attribute specification. -use std::borrow::Cow; use ordered_float::OrderedFloat; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; -use schemars::gen::SchemaGenerator; -use schemars::JsonSchema; -use schemars::schema::{InstanceType, Schema, SchemaObject}; use crate::stability::Stability; @@ -337,7 +334,7 @@ impl Display for EnumEntriesSpec { } /// The different types of values. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] #[serde(rename_all = "snake_case")] #[serde(untagged)] pub enum ValueSpec { @@ -349,30 +346,6 @@ pub enum ValueSpec { String(String), } -// ToDo -// impl JsonSchema for OrderedFloat { -// fn is_referenceable() -> bool { -// false -// } -// -// fn schema_name() -> String { -// "double".to_owned() -// } -// -// fn schema_id() -> Cow<'static, str> { -// Cow::Borrowed("double") -// } -// -// fn json_schema(_: &mut SchemaGenerator) -> Schema { -// SchemaObject { -// instance_type: Some(InstanceType::Number.into()), -// format: (Some("double".to_owned())), -// ..Default::default() -// } -// .into() -// } -// } - /// Implements a human readable display for Value. impl Display for ValueSpec { /// Formats the value. diff --git a/crates/weaver_semconv/src/group.rs b/crates/weaver_semconv/src/group.rs index 9a9b6af6..f6df958e 100644 --- a/crates/weaver_semconv/src/group.rs +++ b/crates/weaver_semconv/src/group.rs @@ -4,8 +4,8 @@ //! A group specification. -use std::fmt::{Display, Formatter}; use schemars::JsonSchema; +use std::fmt::{Display, Formatter}; use serde::{Deserialize, Serialize}; diff --git a/crates/weaver_semconv/src/stability.rs b/crates/weaver_semconv/src/stability.rs index e8dd86c1..bbb42230 100644 --- a/crates/weaver_semconv/src/stability.rs +++ b/crates/weaver_semconv/src/stability.rs @@ -2,9 +2,9 @@ //! Stability specification. +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; -use schemars::JsonSchema; /// The level of stability for a definition. #[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Hash, JsonSchema)] diff --git a/crates/weaver_semconv_gen/src/lib.rs b/crates/weaver_semconv_gen/src/lib.rs index 8a09a9e7..88fbb97c 100644 --- a/crates/weaver_semconv_gen/src/lib.rs +++ b/crates/weaver_semconv_gen/src/lib.rs @@ -12,7 +12,7 @@ use weaver_cache::Cache; use weaver_common::diagnostic::{DiagnosticMessage, DiagnosticMessages}; use weaver_common::error::{format_errors, WeaverError}; use weaver_diff::diff_output; -use weaver_forge::registry::TemplateGroup; +use weaver_forge::registry::ResolvedGroup; use weaver_forge::TemplateEngine; use weaver_resolved_schema::attribute::{Attribute, AttributeRef}; use weaver_resolved_schema::catalog::Catalog; @@ -207,7 +207,7 @@ impl GenerateMarkdownArgs { /// This struct is passed into markdown snippets for generation. #[derive(Serialize)] struct MarkdownSnippetContext { - group: TemplateGroup, + group: ResolvedGroup, snippet_type: SnippetType, tag_filter: Vec, #[serde(skip_serializing_if = "Option::is_none")] @@ -307,7 +307,7 @@ impl SnippetGenerator { .ok_or(Error::GroupNotFound { id: args.id.clone(), }) - .and_then(|g| Ok(TemplateGroup::try_from_resolved(g, self.lookup.catalog())?))?; + .and_then(|g| Ok(ResolvedGroup::try_from_resolved(g, self.lookup.catalog())?))?; // Context is the JSON sent to the jinja template engine. let context = MarkdownSnippetContext { group: group.clone(), diff --git a/crates/weaver_version/Cargo.toml b/crates/weaver_version/Cargo.toml index 429f3ea0..e34bd184 100644 --- a/crates/weaver_version/Cargo.toml +++ b/crates/weaver_version/Cargo.toml @@ -15,5 +15,6 @@ workspace = true serde.workspace = true serde_yaml.workspace = true thiserror.workspace = true +schemars.workspace = true semver = {version = "1.0.22", features = ["serde"]} diff --git a/crates/weaver_version/allowed-external-types.toml b/crates/weaver_version/allowed-external-types.toml index ac42be8c..ee5b8b25 100644 --- a/crates/weaver_version/allowed-external-types.toml +++ b/crates/weaver_version/allowed-external-types.toml @@ -5,4 +5,5 @@ allowed_external_types = [ "serde::ser::Serialize", "serde::de::Deserialize", + "schemars::JsonSchema", ] \ No newline at end of file diff --git a/crates/weaver_version/src/lib.rs b/crates/weaver_version/src/lib.rs index 604a1f6d..e3c71981 100644 --- a/crates/weaver_version/src/lib.rs +++ b/crates/weaver_version/src/lib.rs @@ -2,6 +2,7 @@ //! The specification of the changes to apply to the schema for different versions. +use schemars::JsonSchema; use std::collections::{BTreeMap, HashMap}; use std::fs::File; use std::io::BufReader; @@ -59,14 +60,14 @@ pub enum Error { pub struct Version(semver::Version); /// List of versions with their changes. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(transparent)] pub struct Versions { versions: BTreeMap, } /// An history of changes to apply to the schema for different versions. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct VersionSpec { /// The changes to apply to the metrics specification for a specific version. diff --git a/crates/weaver_version/src/logs_change.rs b/crates/weaver_version/src/logs_change.rs index 3c2cb0e8..0230baeb 100644 --- a/crates/weaver_version/src/logs_change.rs +++ b/crates/weaver_version/src/logs_change.rs @@ -2,11 +2,12 @@ //! Changes to apply to the logs for a specific version. +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::HashMap; /// Changes to apply to the logs for a specific version. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct LogsChange { /// A collection of rename operations to apply to the log attributes. @@ -14,7 +15,7 @@ pub struct LogsChange { } /// A collection of rename operations to apply to the log attributes. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RenameAttributes { /// A collection of rename operations to apply to the log attributes. diff --git a/crates/weaver_version/src/logs_version.rs b/crates/weaver_version/src/logs_version.rs index 96357e66..a6963a53 100644 --- a/crates/weaver_version/src/logs_version.rs +++ b/crates/weaver_version/src/logs_version.rs @@ -3,10 +3,11 @@ //! Logs version. use crate::logs_change::LogsChange; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// Changes to apply to the logs for a specific version. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct LogsVersion { /// Changes to apply to the logs for a specific version. diff --git a/crates/weaver_version/src/metrics_change.rs b/crates/weaver_version/src/metrics_change.rs index fcb7d7b1..db8d65f8 100644 --- a/crates/weaver_version/src/metrics_change.rs +++ b/crates/weaver_version/src/metrics_change.rs @@ -2,11 +2,12 @@ //! Metrics change definitions. +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::HashMap; /// Changes to apply to the metrics for a specific version. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct MetricsChange { /// A collection of rename operations to apply to the metric attributes. @@ -18,7 +19,7 @@ pub struct MetricsChange { } /// A collection of rename operations to apply to the metric attributes. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RenameAttributes { /// A collection of rename operations to apply to the metric attributes. diff --git a/crates/weaver_version/src/metrics_version.rs b/crates/weaver_version/src/metrics_version.rs index da16cc36..8a2c8583 100644 --- a/crates/weaver_version/src/metrics_version.rs +++ b/crates/weaver_version/src/metrics_version.rs @@ -3,10 +3,11 @@ //! Metrics version. use crate::metrics_change::MetricsChange; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// Changes to apply to the metrics for a specific version. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct MetricsVersion { /// Changes to apply to the metrics for a specific version. diff --git a/crates/weaver_version/src/resource_change.rs b/crates/weaver_version/src/resource_change.rs index caf22c30..7e3fbd2e 100644 --- a/crates/weaver_version/src/resource_change.rs +++ b/crates/weaver_version/src/resource_change.rs @@ -2,11 +2,12 @@ //! Changes to apply to the resources for a specific version. +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::HashMap; /// Changes to apply to the resources for a specific version. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct ResourceChange { /// Changes to apply to the resource attributes for a specific version. @@ -14,7 +15,7 @@ pub struct ResourceChange { } /// Changes to apply to the resource attributes for a specific version. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RenameAttributes { /// A collection of rename operations to apply to the resource attributes. diff --git a/crates/weaver_version/src/resource_version.rs b/crates/weaver_version/src/resource_version.rs index f3e4e20f..a559c8cb 100644 --- a/crates/weaver_version/src/resource_version.rs +++ b/crates/weaver_version/src/resource_version.rs @@ -3,10 +3,11 @@ //! Resource version. use crate::resource_change::ResourceChange; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// Changes to apply to the resource for a specific version. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct ResourceVersion { /// Changes to apply to the resource for a specific version. diff --git a/crates/weaver_version/src/spans_change.rs b/crates/weaver_version/src/spans_change.rs index ba2946ca..57050cbd 100644 --- a/crates/weaver_version/src/spans_change.rs +++ b/crates/weaver_version/src/spans_change.rs @@ -2,11 +2,12 @@ //! Spans change specification. +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::HashMap; /// Changes to apply to the spans specification for a specific version. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct SpansChange { /// Changes to apply to the span attributes for a specific version. @@ -14,7 +15,7 @@ pub struct SpansChange { } /// Changes to apply to the span attributes for a specific version. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RenameAttributes { /// A collection of rename operations to apply to the span attributes. diff --git a/crates/weaver_version/src/spans_version.rs b/crates/weaver_version/src/spans_version.rs index 93ed2257..8fcdc6ef 100644 --- a/crates/weaver_version/src/spans_version.rs +++ b/crates/weaver_version/src/spans_version.rs @@ -3,10 +3,11 @@ //! Changes to apply to the spans specification for a specific version. use crate::spans_change::SpansChange; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// Changes to apply to the spans specification for a specific version. -#[derive(Serialize, Deserialize, Debug, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, Default, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct SpansVersion { /// Changes to apply to the spans specification for a specific version. diff --git a/docs/images/dependencies.svg b/docs/images/dependencies.svg index 4dcdd6d6..4a273d14 100644 --- a/docs/images/dependencies.svg +++ b/docs/images/dependencies.svg @@ -4,33 +4,39 @@ - + - + 0 - -weaver_cache + +weaver_cache 1 - -weaver_checker + +weaver_common + + + +0->1 + + 2 - -weaver_common + +weaver_checker - - -1->2 - - + + +2->1 + + @@ -41,92 +47,92 @@ 4 - -weaver_forge + +weaver_forge 9 - -weaver_resolver + +weaver_resolver - + 4->9 - - + + 5 - -weaver_diff + +weaver_diff 6 - -weaver_resolved_schema + +weaver_resolved_schema 7 - -weaver_semconv + +weaver_semconv - + 6->7 - - + + 8 - -weaver_version + +weaver_version - + 6->8 - - + + - - -7->2 - - + + +7->1 + + - + 9->0 - - + + - + 9->5 - - + + - + 9->6 - - + + 10 - -weaver_semconv_gen + +weaver_semconv_gen - + 10->4 - - + + @@ -140,17 +146,17 @@ weaver - - -12->1 - - + + +12->2 + + - + 12->10 - - + + diff --git a/src/registry/generate.rs b/src/registry/generate.rs index 43f6c113..3ad8f487 100644 --- a/src/registry/generate.rs +++ b/src/registry/generate.rs @@ -10,7 +10,7 @@ use weaver_cache::Cache; use weaver_common::diagnostic::DiagnosticMessages; use weaver_common::Logger; use weaver_forge::file_loader::FileSystemFileLoader; -use weaver_forge::registry::TemplateRegistry; +use weaver_forge::registry::ResolvedRegistry; use weaver_forge::{OutputDirective, TemplateEngine}; use weaver_semconv::registry::SemConvRegistry; @@ -87,7 +87,7 @@ pub(crate) fn command( let loader = FileSystemFileLoader::try_new(args.templates.join("registry"), &args.target)?; let engine = TemplateEngine::try_new(loader)?; - let template_registry = TemplateRegistry::try_from_resolved_registry( + let template_registry = ResolvedRegistry::try_from_resolved_registry( schema .registry(registry_id) .expect("Failed to get the registry from the resolved schema"), diff --git a/src/registry/resolve.rs b/src/registry/resolve.rs index 6b0b8fda..738b02a5 100644 --- a/src/registry/resolve.rs +++ b/src/registry/resolve.rs @@ -11,7 +11,7 @@ use crate::DiagnosticArgs; use weaver_cache::Cache; use weaver_common::diagnostic::DiagnosticMessages; use weaver_common::Logger; -use weaver_forge::registry::TemplateRegistry; +use weaver_forge::registry::ResolvedRegistry; use weaver_semconv::registry::SemConvRegistry; use crate::registry::{ @@ -112,7 +112,7 @@ pub(crate) fn command( // The template registry does not include any reference to a shared // catalog of attributes. false => { - let registry = TemplateRegistry::try_from_resolved_registry( + let registry = ResolvedRegistry::try_from_resolved_registry( schema .registry(registry_id) .expect("Failed to get the registry from the resolved schema"), From f4dad7de8247baa110c9cd47dcd025c0f62357ec Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Wed, 29 May 2024 17:39:27 -0700 Subject: [PATCH 3/9] feat(schema): Add new schema command and resolve subcommand. --- src/cli.rs | 3 + src/format.rs | 25 +++++ src/main.rs | 5 + src/registry/check.rs | 4 +- src/registry/generate.rs | 4 +- src/registry/mod.rs | 157 +---------------------------- src/registry/resolve.rs | 98 ++++++------------- src/registry/stats.rs | 5 +- src/registry/update_markdown.rs | 3 +- src/schema/mod.rs | 49 ++++++++++ src/schema/resolve.rs | 168 ++++++++++++++++++++++++++++++++ 11 files changed, 293 insertions(+), 228 deletions(-) create mode 100644 src/format.rs create mode 100644 src/schema/mod.rs create mode 100644 src/schema/resolve.rs diff --git a/src/cli.rs b/src/cli.rs index e1f30fe7..8bf21556 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -4,6 +4,7 @@ use crate::diagnostic::DiagnosticCommand; use crate::registry::RegistryCommand; +use crate::schema::SchemaCommand; use clap::{Parser, Subcommand}; /// Command line arguments. @@ -28,6 +29,8 @@ pub struct Cli { pub enum Commands { /// Manage Semantic Convention Registry Registry(RegistryCommand), + /// Manage Telemetry Schema + Schema(SchemaCommand), /// Manage Diagnostic Messages Diagnostic(DiagnosticCommand), } diff --git a/src/format.rs b/src/format.rs new file mode 100644 index 00000000..96173879 --- /dev/null +++ b/src/format.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Supported output formats for the resolved registry and telemetry schema + +use clap::ValueEnum; +use serde::Serialize; + +/// Supported output formats for the resolved schema +#[derive(Debug, Clone, ValueEnum)] +pub(crate) enum Format { + /// YAML format + Yaml, + /// JSON format + Json, +} + +#[cfg(not(tarpaulin_include))] +pub(crate) fn apply_format(format: &Format, object: &T) -> Result { + match format { + Format::Yaml => serde_yaml::to_string(object) + .map_err(|e| format!("Failed to serialize in Yaml the resolved registry: {:?}", e)), + Format::Json => serde_json::to_string_pretty(object) + .map_err(|e| format!("Failed to serialize in Json the resolved registry: {:?}", e)), + } +} diff --git a/src/main.rs b/src/main.rs index 917482b4..5b355a44 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,10 +15,14 @@ use weaver_forge::{OutputDirective, TemplateEngine}; use crate::cli::{Cli, Commands}; use crate::diagnostic::DEFAULT_DIAGNOSTIC_TEMPLATES; +use crate::schema::telemetry_schema; mod cli; mod diagnostic; +mod format; mod registry; +mod schema; +mod util; /// Set of parameters used to specify the diagnostic format. #[derive(Args, Debug, Clone)] @@ -90,6 +94,7 @@ fn main() { fn run_command(cli: &Cli, log: impl Logger + Sync + Clone) -> i32 { let cmd_result = match &cli.command { Some(Commands::Registry(params)) => semconv_registry(log.clone(), params), + Some(Commands::Schema(params)) => telemetry_schema(log.clone(), params), Some(Commands::Diagnostic(params)) => diagnostic::diagnostic(log.clone(), params), None => return 0, }; diff --git a/src/registry/check.rs b/src/registry/check.rs index c6bdf828..78a78a86 100644 --- a/src/registry/check.rs +++ b/src/registry/check.rs @@ -2,9 +2,9 @@ //! Check a semantic convention registry. -use crate::registry::{ +use crate::registry::RegistryArgs; +use crate::util::{ check_policies, load_semconv_specs, resolve_semconv_specs, semconv_registry_path_from, - RegistryArgs, }; use crate::DiagnosticArgs; use clap::Args; diff --git a/src/registry/generate.rs b/src/registry/generate.rs index 3ad8f487..731ed015 100644 --- a/src/registry/generate.rs +++ b/src/registry/generate.rs @@ -14,9 +14,9 @@ use weaver_forge::registry::ResolvedRegistry; use weaver_forge::{OutputDirective, TemplateEngine}; use weaver_semconv::registry::SemConvRegistry; -use crate::registry::{ +use crate::registry::RegistryArgs; +use crate::util::{ check_policies, load_semconv_specs, resolve_semconv_specs, semconv_registry_path_from, - RegistryArgs, }; use crate::DiagnosticArgs; diff --git a/src/registry/mod.rs b/src/registry/mod.rs index bf34efdb..72649762 100644 --- a/src/registry/mod.rs +++ b/src/registry/mod.rs @@ -3,23 +3,9 @@ //! Commands to manage a semantic convention registry. use std::fmt::Display; -use std::path::PathBuf; use std::str::FromStr; use clap::{Args, Subcommand}; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; - -use check::RegistryCheckArgs; -use weaver_cache::Cache; -use weaver_checker::Error::{InvalidPolicyFile, PolicyViolation}; -use weaver_checker::{Engine, Error, PolicyStage}; -use weaver_common::diagnostic::DiagnosticMessages; -use weaver_common::error::handle_errors; -use weaver_common::Logger; -use weaver_resolved_schema::ResolvedTelemetrySchema; -use weaver_resolver::SchemaResolver; -use weaver_semconv::registry::SemConvRegistry; -use weaver_semconv::semconv::SemConvSpec; use crate::registry::generate::RegistryGenerateArgs; use crate::registry::resolve::RegistryResolveArgs; @@ -27,6 +13,9 @@ use crate::registry::search::RegistrySearchArgs; use crate::registry::stats::RegistryStatsArgs; use crate::registry::update_markdown::RegistryUpdateMarkdownArgs; use crate::CmdResult; +use check::RegistryCheckArgs; +use weaver_cache::Cache; +use weaver_common::Logger; mod check; mod generate; @@ -169,143 +158,3 @@ pub fn semconv_registry(log: impl Logger + Sync + Clone, command: &RegistryComma ), } } - -/// Convert a `RegistryPath` to a `weaver_semconv::path::RegistryPath`. -pub(crate) fn semconv_registry_path_from( - registry: &RegistryPath, - path: &Option, -) -> weaver_semconv::path::RegistryPath { - match registry { - RegistryPath::Local(path) => weaver_semconv::path::RegistryPath::Local { - path_pattern: path.clone(), - }, - RegistryPath::Url(url) => weaver_semconv::path::RegistryPath::GitUrl { - git_url: url.clone(), - path: path.clone(), - }, - } -} - -/// Load the semantic convention specifications from a registry path. -/// -/// # Arguments -/// -/// * `registry_path` - The path to the semantic convention registry. -/// * `cache` - The cache to use for loading the registry. -/// * `log` - The logger to use for logging messages. -pub(crate) fn load_semconv_specs( - registry_path: &weaver_semconv::path::RegistryPath, - cache: &Cache, - log: impl Logger + Sync + Clone, -) -> Result, weaver_resolver::Error> { - let semconv_specs = SchemaResolver::load_semconv_specs(registry_path, cache)?; - log.success(&format!( - "SemConv registry loaded ({} files)", - semconv_specs.len() - )); - Ok(semconv_specs) -} - -/// Check the policies of a semantic convention registry. -/// -/// # Arguments -/// -/// * `policy_engine` - The pre-configured policy engine to use for checking the policies. -/// * `semconv_specs` - The semantic convention specifications to check. -pub fn check_policy( - policy_engine: &Engine, - semconv_specs: &[(String, SemConvSpec)], -) -> Result<(), Error> { - // Check policies in parallel - let policy_errors = semconv_specs - .par_iter() - .flat_map(|(path, semconv)| { - // Create a local policy engine inheriting the policies - // from the global policy engine - let mut policy_engine = policy_engine.clone(); - let mut errors = vec![]; - - match policy_engine.set_input(semconv) { - Ok(_) => match policy_engine.check(PolicyStage::BeforeResolution) { - Ok(violations) => { - for violation in violations { - errors.push(PolicyViolation { - provenance: path.clone(), - violation, - }); - } - } - Err(e) => errors.push(InvalidPolicyFile { - file: path.to_string(), - error: e.to_string(), - }), - }, - Err(e) => errors.push(InvalidPolicyFile { - file: path.to_string(), - error: e.to_string(), - }), - } - errors - }) - .collect::>(); - - handle_errors(policy_errors)?; - Ok(()) -} - -/// Check the policies of a semantic convention registry. -/// -/// # Arguments -/// -/// * `policies` - The list of policy files to check. -/// * `semconv_specs` - The semantic convention specifications to check. -/// * `logger` - The logger to use for logging messages. -fn check_policies( - registry_path: &weaver_semconv::path::RegistryPath, - cache: &Cache, - policies: &[PathBuf], - semconv_specs: &[(String, SemConvSpec)], - logger: impl Logger + Sync + Clone, -) -> Result<(), DiagnosticMessages> { - let mut engine = Engine::new(); - - // Add policies from the registry - let (registry_path, _) = SchemaResolver::path_to_registry(registry_path, cache)?; - let added_policies_count = engine.add_policies(registry_path.as_path(), "*.rego")?; - - // Add policies from the command line - for policy in policies { - engine.add_policy(policy)?; - } - - if added_policies_count + policies.len() > 0 { - check_policy(&engine, semconv_specs).map_err(|e| { - if let Error::CompoundError(errors) = e { - DiagnosticMessages::from_errors(errors) - } else { - DiagnosticMessages::from_error(e) - } - })?; - logger.success("Policies checked"); - } else { - logger.success("No policy found"); - } - Ok(()) -} - -/// Resolve the semantic convention specifications and return the resolved schema. -/// -/// # Arguments -/// -/// * `registry_id` - The ID of the semantic convention registry. -/// * `semconv_specs` - The semantic convention specifications to resolve. -/// * `logger` - The logger to use for logging messages. -pub(crate) fn resolve_semconv_specs( - registry: &mut SemConvRegistry, - logger: impl Logger + Sync + Clone, -) -> Result { - let resolved_schema = SchemaResolver::resolve_semantic_convention_registry(registry)?; - - logger.success("SemConv registry resolved"); - Ok(resolved_schema) -} diff --git a/src/registry/resolve.rs b/src/registry/resolve.rs index 738b02a5..2009533b 100644 --- a/src/registry/resolve.rs +++ b/src/registry/resolve.rs @@ -4,29 +4,20 @@ use std::path::PathBuf; -use clap::{Args, ValueEnum}; -use serde::Serialize; +use clap::Args; -use crate::DiagnosticArgs; use weaver_cache::Cache; use weaver_common::diagnostic::DiagnosticMessages; use weaver_common::Logger; use weaver_forge::registry::ResolvedRegistry; use weaver_semconv::registry::SemConvRegistry; -use crate::registry::{ +use crate::format::{apply_format, Format}; +use crate::registry::RegistryArgs; +use crate::util::{ check_policies, load_semconv_specs, resolve_semconv_specs, semconv_registry_path_from, - RegistryArgs, }; - -/// Supported output formats for the resolved schema -#[derive(Debug, Clone, ValueEnum)] -enum Format { - /// YAML format - Yaml, - /// JSON format - Json, -} +use crate::DiagnosticArgs; /// Parameters for the `registry resolve` sub-command #[derive(Debug, Args)] @@ -35,10 +26,6 @@ pub struct RegistryResolveArgs { #[command(flatten)] registry: RegistryArgs, - /// Flag to indicate if the shared catalog should be included in the resolved schema - #[arg(long, default_value = "false")] - catalog: bool, - /// Flag to indicate if lineage information should be included in the /// resolved schema (not yet implemented) #[arg(long, default_value = "false")] @@ -103,61 +90,42 @@ pub(crate) fn command( // Serialize the resolved schema and write it // to a file or print it to stdout. - match args.catalog { - // The original resolved schema already includes the catalog. - // So, we just need to serialize it. - true => apply_format(&args.format, &schema) - .map_err(|e| format!("Failed to serialize the registry: {e:?}")), - // Build a template registry from the resolved schema and serialize it. - // The template registry does not include any reference to a shared - // catalog of attributes. - false => { - let registry = ResolvedRegistry::try_from_resolved_registry( - schema - .registry(registry_id) - .expect("Failed to get the registry from the resolved schema"), - schema.catalog(), - ) - .unwrap_or_else(|e| panic!("Failed to create the registry without catalog: {e:?}")); - apply_format(&args.format, ®istry) - .map_err(|e| format!("Failed to serialize the registry: {e:?}")) - } - } - .and_then(|s| { - if let Some(ref path) = args.output { - // Write the resolved registry to a file. - std::fs::write(path, s) - .map_err(|e| format!("Failed to write the resolved registry to file: {e:?}")) - } else { - // Print the resolved registry to stdout. - println!("{}", s); - Ok(()) - } - }) - .unwrap_or_else(|e| { - // Capture all the errors - panic!("{}", e); - }); + let registry = ResolvedRegistry::try_from_resolved_registry( + schema + .registry(registry_id) + .expect("Failed to get the registry from the resolved schema"), + schema.catalog(), + ) + .unwrap_or_else(|e| panic!("Failed to create the registry without catalog: {e:?}")); + + apply_format(&args.format, ®istry) + .map_err(|e| format!("Failed to serialize the registry: {e:?}")) + .and_then(|s| { + if let Some(ref path) = args.output { + // Write the resolved registry to a file. + std::fs::write(path, s) + .map_err(|e| format!("Failed to write the resolved registry to file: {e:?}")) + } else { + // Print the resolved registry to stdout. + println!("{}", s); + Ok(()) + } + }) + .unwrap_or_else(|e| { + // Capture all the errors + panic!("{}", e); + }); Ok(()) } -#[cfg(not(tarpaulin_include))] -fn apply_format(format: &Format, object: &T) -> Result { - match format { - Format::Yaml => serde_yaml::to_string(object) - .map_err(|e| format!("Failed to serialize in Yaml the resolved registry: {:?}", e)), - Format::Json => serde_json::to_string_pretty(object) - .map_err(|e| format!("Failed to serialize in Json the resolved registry: {:?}", e)), - } -} - #[cfg(test)] mod tests { use weaver_common::TestLogger; use crate::cli::{Cli, Commands}; - use crate::registry::resolve::{Format, RegistryResolveArgs}; + use crate::format::Format; + use crate::registry::resolve::RegistryResolveArgs; use crate::registry::{RegistryArgs, RegistryCommand, RegistryPath, RegistrySubCommand}; use crate::run_command; @@ -175,7 +143,6 @@ mod tests { ), registry_git_sub_dir: None, }, - catalog: false, lineage: true, output: None, format: Format::Yaml, @@ -202,7 +169,6 @@ mod tests { ), registry_git_sub_dir: None, }, - catalog: false, lineage: true, output: None, format: Format::Json, diff --git a/src/registry/stats.rs b/src/registry/stats.rs index 8cba61ca..cbc8a3a4 100644 --- a/src/registry/stats.rs +++ b/src/registry/stats.rs @@ -2,9 +2,8 @@ //! Compute stats on a semantic convention registry. -use crate::registry::{ - load_semconv_specs, resolve_semconv_specs, semconv_registry_path_from, RegistryArgs, -}; +use crate::registry::RegistryArgs; +use crate::util::{load_semconv_specs, resolve_semconv_specs, semconv_registry_path_from}; use crate::DiagnosticArgs; use clap::Args; use weaver_cache::Cache; diff --git a/src/registry/update_markdown.rs b/src/registry/update_markdown.rs index 1242da18..b919bea6 100644 --- a/src/registry/update_markdown.rs +++ b/src/registry/update_markdown.rs @@ -3,7 +3,8 @@ //! Update markdown files that contain markers indicating the templates used to //! update the specified sections. -use crate::registry::{semconv_registry_path_from, RegistryArgs}; +use crate::registry::RegistryArgs; +use crate::util::semconv_registry_path_from; use crate::DiagnosticArgs; use clap::Args; use weaver_cache::Cache; diff --git a/src/schema/mod.rs b/src/schema/mod.rs new file mode 100644 index 00000000..fd114184 --- /dev/null +++ b/src/schema/mod.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Commands to manage a telemetry schema. + +mod resolve; + +use crate::schema::resolve::SchemaResolveArgs; +use crate::CmdResult; +use clap::{Args, Subcommand}; +use weaver_cache::Cache; +use weaver_common::Logger; + +/// Parameters for the `registry` command +#[derive(Debug, Args)] +pub struct SchemaCommand { + /// Define the sub-commands for the `schema` command + #[clap(subcommand)] + pub command: SchemaSubCommand, +} + +/// Sub-commands to manage a `schema`. +#[derive(Debug, Subcommand)] +#[clap(verbatim_doc_comment)] +pub enum SchemaSubCommand { + /// Resolves a telemetry schema. + /// + /// Rego policies present in the registry or specified using -p or --policy will be automatically validated by the policy engine before the artifact generation phase. + /// + /// Note: The `-d` and `--registry-git-sub-dir` options are only used when the registry is a Git URL otherwise these options are ignored. + /// + /// The process exits with a code of 0 if the resolution is successful. + #[clap(verbatim_doc_comment)] + Resolve(SchemaResolveArgs), +} + +/// Manage a telemetry schema and return the exit code. +pub fn telemetry_schema(log: impl Logger + Sync + Clone, command: &SchemaCommand) -> CmdResult { + let cache = match Cache::try_new() { + Ok(cache) => cache, + Err(e) => return CmdResult::new(Err(e.into()), None), + }; + + match &command.command { + SchemaSubCommand::Resolve(args) => CmdResult::new( + resolve::command(log.clone(), &cache, args), + Some(args.diagnostic.clone()), + ), + } +} diff --git a/src/schema/resolve.rs b/src/schema/resolve.rs new file mode 100644 index 00000000..9e2a38f3 --- /dev/null +++ b/src/schema/resolve.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Resolve a telemetry schema. + +use crate::format::Format; +use crate::registry::RegistryArgs; +use crate::util::{ + check_policies, load_semconv_specs, resolve_semconv_specs, semconv_registry_path_from, +}; +use crate::{format, DiagnosticArgs}; +use clap::Args; +use std::path::PathBuf; +use weaver_cache::Cache; +use weaver_common::diagnostic::DiagnosticMessages; +use weaver_common::Logger; +use weaver_semconv::registry::SemConvRegistry; + +/// Parameters for the `schema resolve` sub-command +#[derive(Debug, Args)] +pub struct SchemaResolveArgs { + /// Parameters to specify the semantic convention registry + #[command(flatten)] + registry: RegistryArgs, + + /// Output file to write the resolved schema to + /// If not specified, the resolved schema is printed to stdout + #[arg(short, long)] + output: Option, + + /// Output format for the resolved schema + /// If not specified, the resolved schema is printed in YAML format + /// Supported formats: yaml, json + /// Default format: yaml + /// Example: `--format json` + #[arg(short, long, default_value = "yaml")] + format: Format, + + /// Optional list of policy files to check against the files of the semantic + /// convention registry. + #[arg(short = 'p', long = "policy")] + pub policies: Vec, + + /// Skip the policy checks. + #[arg(long, default_value = "false")] + pub skip_policies: bool, + + /// Parameters to specify the diagnostic format. + #[command(flatten)] + pub diagnostic: DiagnosticArgs, +} + +/// Resolve a telemetry schema and write the resolved schema to a +/// file or print it to stdout. +#[cfg(not(tarpaulin_include))] +pub(crate) fn command( + logger: impl Logger + Sync + Clone, + cache: &Cache, + args: &SchemaResolveArgs, +) -> Result<(), DiagnosticMessages> { + logger.loading(&format!("Resolving schema `{}`", args.registry.registry)); + + let registry_id = "default"; + let registry_path = + semconv_registry_path_from(&args.registry.registry, &args.registry.registry_git_sub_dir); + + // Load the semantic convention registry into a local cache. + let semconv_specs = load_semconv_specs(®istry_path, cache, logger.clone())?; + + if !args.skip_policies { + check_policies( + ®istry_path, + cache, + &args.policies, + &semconv_specs, + logger.clone(), + )?; + } + + let mut registry = SemConvRegistry::from_semconv_specs(registry_id, semconv_specs); + let schema = resolve_semconv_specs(&mut registry, logger.clone())?; + + // Serialize the resolved schema and write it + // to a file or print it to stdout. + format::apply_format(&args.format, &schema) + .map_err(|e| format!("Failed to serialize the registry: {e:?}")) + .and_then(|s| { + if let Some(ref path) = args.output { + // Write the resolved registry to a file. + std::fs::write(path, s) + .map_err(|e| format!("Failed to write the resolved registry to file: {e:?}")) + } else { + // Print the resolved registry to stdout. + println!("{}", s); + Ok(()) + } + }) + .unwrap_or_else(|e| { + // Capture all the errors + panic!("{}", e); + }); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use weaver_common::TestLogger; + + use crate::cli::{Cli, Commands}; + use crate::format::Format; + use crate::registry::{RegistryArgs, RegistryPath}; + use crate::run_command; + use crate::schema::resolve::SchemaResolveArgs; + use crate::schema::{SchemaCommand, SchemaSubCommand}; + + #[test] + fn test_schema_resolve() { + let logger = TestLogger::new(); + let cli = Cli { + debug: 0, + quiet: false, + command: Some(Commands::Schema(SchemaCommand { + command: SchemaSubCommand::Resolve(SchemaResolveArgs { + registry: RegistryArgs { + registry: RegistryPath::Local( + "crates/weaver_codegen_test/semconv_registry/".to_owned(), + ), + registry_git_sub_dir: None, + }, + output: None, + format: Format::Yaml, + policies: vec![], + skip_policies: true, + diagnostic: Default::default(), + }), + })), + }; + + let exit_code = run_command(&cli, logger.clone()); + // The command should succeed. + assert_eq!(exit_code, 0); + + // Now, let's run the command again with the policy checks enabled. + let cli = Cli { + debug: 0, + quiet: false, + command: Some(Commands::Schema(SchemaCommand { + command: SchemaSubCommand::Resolve(SchemaResolveArgs { + registry: RegistryArgs { + registry: RegistryPath::Local( + "crates/weaver_codegen_test/semconv_registry/".to_owned(), + ), + registry_git_sub_dir: None, + }, + output: None, + format: Format::Json, + policies: vec![], + skip_policies: false, + diagnostic: Default::default(), + }), + })), + }; + + let exit_code = run_command(&cli, logger); + // The command should exit with an error code. + assert_eq!(exit_code, 1); + } +} From 1259bed17aa5079ed433032fd1f77b20c30538b9 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Wed, 29 May 2024 17:44:03 -0700 Subject: [PATCH 4/9] feat(schema): Add new schema command and resolve subcommand. --- src/util.rs | 158 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 src/util.rs diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 00000000..df0abca0 --- /dev/null +++ b/src/util.rs @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Set of utility functions to resolve a semantic convention registry and check policies. +//! This module is used by the `schema` and `registry` commands. + +use crate::registry::RegistryPath; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::path::PathBuf; +use weaver_cache::Cache; +use weaver_checker::Error::{InvalidPolicyFile, PolicyViolation}; +use weaver_checker::{Engine, Error, PolicyStage}; +use weaver_common::diagnostic::DiagnosticMessages; +use weaver_common::error::handle_errors; +use weaver_common::Logger; +use weaver_resolved_schema::ResolvedTelemetrySchema; +use weaver_resolver::SchemaResolver; +use weaver_semconv::registry::SemConvRegistry; +use weaver_semconv::semconv::SemConvSpec; + +/// Convert a `RegistryPath` to a `weaver_semconv::path::RegistryPath`. +pub(crate) fn semconv_registry_path_from( + registry: &RegistryPath, + path: &Option, +) -> weaver_semconv::path::RegistryPath { + match registry { + RegistryPath::Local(path) => weaver_semconv::path::RegistryPath::Local { + path_pattern: path.clone(), + }, + RegistryPath::Url(url) => weaver_semconv::path::RegistryPath::GitUrl { + git_url: url.clone(), + path: path.clone(), + }, + } +} + +/// Load the semantic convention specifications from a registry path. +/// +/// # Arguments +/// +/// * `registry_path` - The path to the semantic convention registry. +/// * `cache` - The cache to use for loading the registry. +/// * `log` - The logger to use for logging messages. +pub(crate) fn load_semconv_specs( + registry_path: &weaver_semconv::path::RegistryPath, + cache: &Cache, + log: impl Logger + Sync + Clone, +) -> Result, weaver_resolver::Error> { + let semconv_specs = SchemaResolver::load_semconv_specs(registry_path, cache)?; + log.success(&format!( + "SemConv registry loaded ({} files)", + semconv_specs.len() + )); + Ok(semconv_specs) +} + +/// Check the policies of a semantic convention registry. +/// +/// # Arguments +/// +/// * `policy_engine` - The pre-configured policy engine to use for checking the policies. +/// * `semconv_specs` - The semantic convention specifications to check. +pub(crate) fn check_policy( + policy_engine: &Engine, + semconv_specs: &[(String, SemConvSpec)], +) -> Result<(), Error> { + // Check policies in parallel + let policy_errors = semconv_specs + .par_iter() + .flat_map(|(path, semconv)| { + // Create a local policy engine inheriting the policies + // from the global policy engine + let mut policy_engine = policy_engine.clone(); + let mut errors = vec![]; + + match policy_engine.set_input(semconv) { + Ok(_) => match policy_engine.check(PolicyStage::BeforeResolution) { + Ok(violations) => { + for violation in violations { + errors.push(PolicyViolation { + provenance: path.clone(), + violation, + }); + } + } + Err(e) => errors.push(InvalidPolicyFile { + file: path.to_string(), + error: e.to_string(), + }), + }, + Err(e) => errors.push(InvalidPolicyFile { + file: path.to_string(), + error: e.to_string(), + }), + } + errors + }) + .collect::>(); + + handle_errors(policy_errors)?; + Ok(()) +} + +/// Check the policies of a semantic convention registry. +/// +/// # Arguments +/// +/// * `policies` - The list of policy files to check. +/// * `semconv_specs` - The semantic convention specifications to check. +/// * `logger` - The logger to use for logging messages. +pub(crate) fn check_policies( + registry_path: &weaver_semconv::path::RegistryPath, + cache: &Cache, + policies: &[PathBuf], + semconv_specs: &[(String, SemConvSpec)], + logger: impl Logger + Sync + Clone, +) -> Result<(), DiagnosticMessages> { + let mut engine = Engine::new(); + + // Add policies from the registry + let (registry_path, _) = SchemaResolver::path_to_registry(registry_path, cache)?; + let added_policies_count = engine.add_policies(registry_path.as_path(), "*.rego")?; + + // Add policies from the command line + for policy in policies { + engine.add_policy(policy)?; + } + + if added_policies_count + policies.len() > 0 { + check_policy(&engine, semconv_specs).map_err(|e| { + if let Error::CompoundError(errors) = e { + DiagnosticMessages::from_errors(errors) + } else { + DiagnosticMessages::from_error(e) + } + })?; + logger.success("Policies checked"); + } else { + logger.success("No policy found"); + } + Ok(()) +} + +/// Resolve the semantic convention specifications and return the resolved schema. +/// +/// # Arguments +/// +/// * `registry_id` - The ID of the semantic convention registry. +/// * `semconv_specs` - The semantic convention specifications to resolve. +/// * `logger` - The logger to use for logging messages. +pub(crate) fn resolve_semconv_specs( + registry: &mut SemConvRegistry, + logger: impl Logger + Sync + Clone, +) -> Result { + let resolved_schema = SchemaResolver::resolve_semantic_convention_registry(registry)?; + + logger.success("SemConv registry resolved"); + Ok(resolved_schema) +} From 70c45d948da2fd02d99ca89963524dcb75e60587 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Thu, 30 May 2024 13:51:47 -0700 Subject: [PATCH 5/9] feat(json-schema): Add command to output the JSON schema of a ResolvedRegistry object. --- Cargo.lock | 5 +- Cargo.toml | 1 + crates/weaver_common/src/in_memory.rs | 5 + crates/weaver_common/src/lib.rs | 57 ++++++++++- crates/weaver_common/src/quiet.rs | 5 + src/diagnostic/init.rs | 17 ++-- src/main.rs | 66 +++++++++---- src/registry/check.rs | 17 ++-- src/registry/generate.rs | 17 ++-- src/registry/json_schema.rs | 137 ++++++++++++++++++++++++++ src/registry/mod.rs | 11 +++ src/registry/resolve.rs | 20 ++-- src/registry/stats.rs | 9 +- src/registry/update_markdown.rs | 13 ++- src/schema/resolve.rs | 17 ++-- 15 files changed, 330 insertions(+), 67 deletions(-) create mode 100644 src/registry/json_schema.rs diff --git a/Cargo.lock b/Cargo.lock index ade2fdb6..d548b4fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3373,9 +3373,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -3760,6 +3760,7 @@ dependencies = [ "include_dir", "miette", "rayon", + "schemars", "serde", "serde_json", "serde_yaml", diff --git a/Cargo.toml b/Cargo.toml index 173a712f..be733843 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ walkdir.workspace = true include_dir.workspace = true thiserror.workspace = true miette.workspace = true +schemars.workspace = true rayon = "1.10.0" diff --git a/crates/weaver_common/src/in_memory.rs b/crates/weaver_common/src/in_memory.rs index 19d8a6c7..eccefef4 100644 --- a/crates/weaver_common/src/in_memory.rs +++ b/crates/weaver_common/src/in_memory.rs @@ -151,4 +151,9 @@ impl crate::Logger for Logger { .expect("Failed to lock messages") .push(LogMessage::Log(message.to_owned())); } + + /// Mute all the messages except for the warnings and errors. + fn mute(&self) { + // We do not mute the logger in this implementation. + } } diff --git a/crates/weaver_common/src/lib.rs b/crates/weaver_common/src/lib.rs index c2e74b1e..b1a830e4 100644 --- a/crates/weaver_common/src/lib.rs +++ b/crates/weaver_common/src/lib.rs @@ -47,6 +47,9 @@ pub trait Logger { /// Logs a message without icon. fn log(&self, message: &str); + + /// Mute all the messages except for the warnings and errors. + fn mute(&self); } /// A generic logger that can be used to log messages to the console. @@ -55,6 +58,7 @@ pub trait Logger { pub struct ConsoleLogger { logger: Arc>>, debug_level: u8, + mute: Arc>, } impl ConsoleLogger { @@ -64,6 +68,7 @@ impl ConsoleLogger { ConsoleLogger { logger: Arc::new(Mutex::new(paris::Logger::new())), debug_level, + mute: Arc::new(Mutex::new(false)), } } } @@ -71,7 +76,8 @@ impl ConsoleLogger { impl Logger for ConsoleLogger { /// Logs an trace message (only with debug enabled). fn trace(&self, message: &str) { - if self.debug_level > 0 { + let mute = *self.mute.lock().expect("Failed to lock mute"); + if self.debug_level > 0 && !mute { _ = self .logger .lock() @@ -82,6 +88,10 @@ impl Logger for ConsoleLogger { /// Logs an info message. fn info(&self, message: &str) { + if *self.mute.lock().expect("Failed to lock mute") { + return; + } + _ = self .logger .lock() @@ -109,6 +119,10 @@ impl Logger for ConsoleLogger { /// Logs a success message. fn success(&self, message: &str) { + if *self.mute.lock().expect("Failed to lock mute") { + return; + } + _ = self .logger .lock() @@ -118,6 +132,10 @@ impl Logger for ConsoleLogger { /// Logs a newline. fn newline(&self, count: usize) { + if *self.mute.lock().expect("Failed to lock mute") { + return; + } + _ = self .logger .lock() @@ -127,6 +145,10 @@ impl Logger for ConsoleLogger { /// Indents the logger. fn indent(&self, count: usize) { + if *self.mute.lock().expect("Failed to lock mute") { + return; + } + _ = self .logger .lock() @@ -136,11 +158,19 @@ impl Logger for ConsoleLogger { /// Stops a loading message. fn done(&self) { + if *self.mute.lock().expect("Failed to lock mute") { + return; + } + _ = self.logger.lock().expect("Failed to lock logger").done(); } /// Adds a style to the logger. fn add_style(&self, name: &str, styles: Vec<&'static str>) -> &Self { + if *self.mute.lock().expect("Failed to lock mute") { + return self; + } + _ = self .logger .lock() @@ -151,6 +181,10 @@ impl Logger for ConsoleLogger { /// Logs a loading message with a spinner. fn loading(&self, message: &str) { + if *self.mute.lock().expect("Failed to lock mute") { + return; + } + _ = self .logger .lock() @@ -160,18 +194,31 @@ impl Logger for ConsoleLogger { /// Forces the logger to not print a newline for the next message. fn same(&self) -> &Self { + if *self.mute.lock().expect("Failed to lock mute") { + return self; + } + _ = self.logger.lock().expect("Failed to lock logger").same(); self } /// Logs a message without icon. fn log(&self, message: &str) { + if *self.mute.lock().expect("Failed to lock mute") { + return; + } + _ = self .logger .lock() .expect("Failed to lock logger") .log(message); } + + /// Mute all the messages except for the warnings and errors. + fn mute(&self) { + *self.mute.lock().expect("Failed to lock mute") = true; + } } /// A logger that does not log anything. @@ -226,6 +273,9 @@ impl Logger for NullLogger { /// Logs a message without icon. fn log(&self, _: &str) {} + + /// Mute all the messages except for the warnings and errors. + fn mute(&self) {} } /// A logger that can be used in unit or integration tests. @@ -363,4 +413,9 @@ impl Logger for TestLogger { .expect("Failed to lock logger") .log(message); } + + /// Mute all the messages except for the warnings and errors. + fn mute(&self) { + // We do not need to mute the logger in the tests. + } } diff --git a/crates/weaver_common/src/quiet.rs b/crates/weaver_common/src/quiet.rs index b549c90a..f5165753 100644 --- a/crates/weaver_common/src/quiet.rs +++ b/crates/weaver_common/src/quiet.rs @@ -78,4 +78,9 @@ impl Logger for QuietLogger { /// Logs a message without icon. fn log(&self, _message: &str) {} + + /// Mute all the messages except for the warnings and errors. + fn mute(&self) { + // Do nothing + } } diff --git a/src/diagnostic/init.rs b/src/diagnostic/init.rs index 6b044796..8be9bdc4 100644 --- a/src/diagnostic/init.rs +++ b/src/diagnostic/init.rs @@ -3,7 +3,7 @@ //! Initializes a `diagnostic_templates` directory to define or override diagnostic output formats. use crate::diagnostic::{Error, DEFAULT_DIAGNOSTIC_TEMPLATES}; -use crate::DiagnosticArgs; +use crate::{DiagnosticArgs, ExitDirectives}; use clap::Args; use include_dir::DirEntry; use std::fs; @@ -32,7 +32,7 @@ pub struct DiagnosticInitArgs { pub(crate) fn command( logger: impl Logger + Sync + Clone, args: &DiagnosticInitArgs, -) -> Result<(), DiagnosticMessages> { +) -> Result { extract(args.diagnostic_templates_dir.clone(), &args.target).map_err(|e| { Error::InitDiagnosticError { path: args.diagnostic_templates_dir.clone(), @@ -44,7 +44,10 @@ pub(crate) fn command( "Diagnostic templates initialized at {:?}", args.diagnostic_templates_dir )); - Ok(()) + Ok(ExitDirectives { + exit_code: 0, + quiet_mode: false, + }) } /// Extracts the diagnostic templates to the specified path for the given target. @@ -104,9 +107,9 @@ mod tests { })), }; - let exit_code = run_command(&cli, logger.clone()); + let exit_directive = run_command(&cli, logger.clone()); // The command should succeed. - assert_eq!(exit_code, 0); + assert_eq!(exit_directive.exit_code, 0); // Check the presence of 3 subdirectories in the temp_output directory let subdirs = fs::read_dir(&temp_output).unwrap().count(); @@ -128,9 +131,9 @@ mod tests { })), }; - let exit_code = run_command(&cli, logger.clone()); + let exit_directive = run_command(&cli, logger.clone()); // The command should succeed. - assert_eq!(exit_code, 0); + assert_eq!(exit_directive.exit_code, 0); // Check the presence of 3 subdirectories in the temp_output directory let subdirs = fs::read_dir(&temp_output).unwrap().count(); diff --git a/src/main.rs b/src/main.rs index 5b355a44..e7cbfe94 100644 --- a/src/main.rs +++ b/src/main.rs @@ -49,14 +49,23 @@ impl Default for DiagnosticArgs { /// Result of a command execution. #[derive(Debug)] pub(crate) struct CmdResult { - pub(crate) command_result: Result<(), DiagnosticMessages>, + pub(crate) command_result: Result, pub(crate) diagnostic_args: Option, } +/// Exit directives. +#[derive(Debug, Clone)] +pub(crate) struct ExitDirectives { + /// Exit code. + exit_code: i32, + /// Quiet mode. + quiet_mode: bool, +} + impl CmdResult { /// Create a new command result. pub(crate) fn new( - command_result: Result<(), DiagnosticMessages>, + command_result: Result, diagnostic_args: Option, ) -> Self { Self { @@ -71,7 +80,7 @@ fn main() { let cli = Cli::parse(); let start = std::time::Instant::now(); - let exit_code = if cli.quiet { + let exit_directives = if cli.quiet { let log = QuietLogger::new(); run_command(&cli, log) } else { @@ -79,37 +88,53 @@ fn main() { run_command(&cli, log) }; - if !cli.quiet { + if !cli.quiet && !exit_directives.quiet_mode { let elapsed = start.elapsed(); - println!("Total execution time: {:?}s", elapsed.as_secs_f64()); + println!("\nTotal execution time: {:?}s", elapsed.as_secs_f64()); } // Exit the process with the exit code provided by the `run_command` function. #[allow(clippy::exit)] - std::process::exit(exit_code); + std::process::exit(exit_directives.exit_code); } -/// Run the command specified by the CLI arguments and return the exit code. +/// Run the command specified by the CLI arguments and return the exit directives. #[cfg(not(tarpaulin_include))] -fn run_command(cli: &Cli, log: impl Logger + Sync + Clone) -> i32 { +fn run_command(cli: &Cli, log: impl Logger + Sync + Clone) -> ExitDirectives { let cmd_result = match &cli.command { Some(Commands::Registry(params)) => semconv_registry(log.clone(), params), Some(Commands::Schema(params)) => telemetry_schema(log.clone(), params), Some(Commands::Diagnostic(params)) => diagnostic::diagnostic(log.clone(), params), - None => return 0, + None => { + return ExitDirectives { + exit_code: 0, + quiet_mode: false, + } + } }; process_diagnostics(cmd_result, log.clone()) } -/// Render the diagnostic messages based on the diagnostic configuration and return the exit code -/// based on the diagnostic messages. -fn process_diagnostics(cmd_result: CmdResult, logger: impl Logger + Sync + Clone) -> i32 { +/// Render the diagnostic messages based on the diagnostic configuration and return the exit +/// directives based on the diagnostic messages and the CmdResult quiet mode. +fn process_diagnostics( + cmd_result: CmdResult, + logger: impl Logger + Sync + Clone, +) -> ExitDirectives { let diagnostic_args = if let Some(diagnostic_args) = cmd_result.diagnostic_args { diagnostic_args } else { DiagnosticArgs::default() // Default diagnostic arguments; }; + let mut exit_directives = if let Ok(exit_directives) = &cmd_result.command_result { + exit_directives.clone() + } else { + ExitDirectives { + exit_code: 0, + quiet_mode: false, + } + }; if let Err(diagnostic_messages) = cmd_result.command_result { let loader = EmbeddedFileLoader::try_new( @@ -132,22 +157,21 @@ fn process_diagnostics(cmd_result: CmdResult, logger: impl Logger + Sync + Clone "Failed to render the diagnostic messages. Error: {}", e )); - return 1; + exit_directives.exit_code = 1; + return exit_directives; } } } Err(e) => { logger.error(&format!("Failed to create the template engine to render the diagnostic messages. Error: {}", e)); - return 1; + exit_directives.exit_code = 1; + return exit_directives; } } - return if diagnostic_messages.has_error() { - 1 - } else { - 0 - }; + if diagnostic_messages.has_error() { + exit_directives.exit_code = 1; + } } - // Return 0 if there are no diagnostic messages - 0 + exit_directives } diff --git a/src/registry/check.rs b/src/registry/check.rs index 78a78a86..13d556b8 100644 --- a/src/registry/check.rs +++ b/src/registry/check.rs @@ -6,7 +6,7 @@ use crate::registry::RegistryArgs; use crate::util::{ check_policies, load_semconv_specs, resolve_semconv_specs, semconv_registry_path_from, }; -use crate::DiagnosticArgs; +use crate::{DiagnosticArgs, ExitDirectives}; use clap::Args; use std::path::PathBuf; use weaver_cache::Cache; @@ -41,7 +41,7 @@ pub(crate) fn command( logger: impl Logger + Sync + Clone, cache: &Cache, args: &RegistryCheckArgs, -) -> Result<(), DiagnosticMessages> { +) -> Result { logger.loading(&format!("Checking registry `{}`", args.registry.registry)); let registry_id = "default"; @@ -65,7 +65,10 @@ pub(crate) fn command( let mut registry = SemConvRegistry::from_semconv_specs(registry_id, semconv_specs); _ = resolve_semconv_specs(&mut registry, logger.clone())?; - Ok(()) + Ok(ExitDirectives { + exit_code: 0, + quiet_mode: false, + }) } #[cfg(test)] @@ -98,9 +101,9 @@ mod tests { })), }; - let exit_code = run_command(&cli, logger.clone()); + let exit_directive = run_command(&cli, logger.clone()); // The command should succeed. - assert_eq!(exit_code, 0); + assert_eq!(exit_directive.exit_code, 0); // Now, let's run the command again with the policy checks enabled. let cli = Cli { @@ -121,8 +124,8 @@ mod tests { })), }; - let exit_code = run_command(&cli, logger); + let exit_directive = run_command(&cli, logger); // The command should exit with an error code. - assert_eq!(exit_code, 1); + assert_eq!(exit_directive.exit_code, 1); } } diff --git a/src/registry/generate.rs b/src/registry/generate.rs index 731ed015..f7bc91d5 100644 --- a/src/registry/generate.rs +++ b/src/registry/generate.rs @@ -18,7 +18,7 @@ use crate::registry::RegistryArgs; use crate::util::{ check_policies, load_semconv_specs, resolve_semconv_specs, semconv_registry_path_from, }; -use crate::DiagnosticArgs; +use crate::{DiagnosticArgs, ExitDirectives}; /// Parameters for the `registry generate` sub-command #[derive(Debug, Args)] @@ -59,7 +59,7 @@ pub(crate) fn command( logger: impl Logger + Sync + Clone, cache: &Cache, args: &RegistryGenerateArgs, -) -> Result<(), DiagnosticMessages> { +) -> Result { logger.loading(&format!( "Generating artifacts for the registry `{}`", args.registry.registry @@ -102,7 +102,10 @@ pub(crate) fn command( )?; logger.success("Artifacts generated successfully"); - Ok(()) + Ok(ExitDirectives { + exit_code: 0, + quiet_mode: false, + }) } #[cfg(test)] @@ -145,9 +148,9 @@ mod tests { })), }; - let exit_code = run_command(&cli, logger.clone()); + let exit_directive = run_command(&cli, logger.clone()); // The command should succeed. - assert_eq!(exit_code, 0); + assert_eq!(exit_directive.exit_code, 0); // Hashset containing recursively all the relative paths of rust files in the // output directory. @@ -212,8 +215,8 @@ mod tests { })), }; - let exit_code = run_command(&cli, logger); + let exit_directive = run_command(&cli, logger); // The command should exit with an error code. - assert_eq!(exit_code, 1); + assert_eq!(exit_directive.exit_code, 1); } } diff --git a/src/registry/json_schema.rs b/src/registry/json_schema.rs new file mode 100644 index 00000000..2a3653aa --- /dev/null +++ b/src/registry/json_schema.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Generate the JSON Schema of the resolved registry documents consumed by the template generator +//! and the policy engine. + +use crate::{DiagnosticArgs, ExitDirectives}; +use clap::Args; +use miette::Diagnostic; +use schemars::schema_for; +use serde::Serialize; +use serde_json::to_string_pretty; +use std::path::PathBuf; +use weaver_cache::Cache; +use weaver_common::diagnostic::{DiagnosticMessage, DiagnosticMessages}; +use weaver_common::Logger; +use weaver_forge::registry::ResolvedRegistry; + +/// Parameters for the `registry json-schema` sub-command +#[derive(Debug, Args)] +pub struct RegistryJsonSchemaArgs { + /// Output file to write the JSON schema to + /// If not specified, the JSON schema is printed to stdout + #[arg(short, long)] + output: Option, + + /// Parameters to specify the diagnostic format. + #[command(flatten)] + pub diagnostic: DiagnosticArgs, +} + +/// An error that can occur while generating a JSON Schema. +#[derive(thiserror::Error, Debug, Clone, PartialEq, Serialize, Diagnostic)] +#[non_exhaustive] +pub enum Error { + /// The serialization of the JSON schema failed. + #[error("The serialization of the JSON schema failed. Error: {error}")] + SerializationError { + /// The error that occurred. + error: String, + }, + + /// Writing to the file failed. + #[error("Writing to the file ‘{file}’ failed for the following reason: {error}")] + WriteError { + /// The path to the output file. + file: PathBuf, + /// The error that occurred. + error: String, + }, +} + +impl From for DiagnosticMessages { + fn from(error: Error) -> Self { + DiagnosticMessages::new(vec![DiagnosticMessage::new(error)]) + } +} + +/// Generate the JSON Schema of a ResolvedRegistry and write the JSON schema to a +/// file or print it to stdout. +#[cfg(not(tarpaulin_include))] +pub(crate) fn command( + logger: impl Logger + Sync + Clone, + _cache: &Cache, + args: &RegistryJsonSchemaArgs, +) -> Result { + let json_schema = schema_for!(ResolvedRegistry); + + let json_schema_str = + to_string_pretty(&json_schema).map_err(|e| Error::SerializationError { + error: e.to_string(), + })?; + + if let Some(output) = &args.output { + logger.loading(&format!("Writing JSON schema to `{}`", output.display())); + std::fs::write(output, json_schema_str).map_err(|e| Error::WriteError { + file: output.clone(), + error: e.to_string(), + })?; + } else { + logger.log(&json_schema_str); + } + + Ok(ExitDirectives { + exit_code: 0, + quiet_mode: args.output.is_none(), + }) +} + +#[cfg(test)] +mod tests { + use weaver_common::in_memory; + use weaver_common::in_memory::LogMessage; + + use crate::cli::{Cli, Commands}; + use crate::registry::json_schema::RegistryJsonSchemaArgs; + use crate::registry::{RegistryCommand, RegistrySubCommand}; + use crate::run_command; + + #[test] + fn test_registry_json_schema() { + let logger = in_memory::Logger::new(0); + let cli = Cli { + debug: 0, + quiet: false, + command: Some(Commands::Registry(RegistryCommand { + command: RegistrySubCommand::JsonSchema(RegistryJsonSchemaArgs { + output: None, + diagnostic: Default::default(), + }), + })), + }; + + let exit_directive = run_command(&cli, logger.clone()); + // The command should succeed. + assert_eq!(exit_directive.exit_code, 0); + + // We should have a single log message with the JSON schema. + let messages = logger.messages(); + assert_eq!(messages.len(), 1); + + let message = &messages[0]; + if let LogMessage::Log(log) = message { + let value = + serde_json::from_str::(log).expect("Failed to parse JSON"); + let definitions = value + .as_object() + .expect("Expected a JSON object") + .get("definitions"); + assert!( + definitions.is_some(), + "Expected a 'definitions' key in the JSON schema" + ); + } else { + panic!("Expected a log message, but got: {:?}", message); + } + } +} diff --git a/src/registry/mod.rs b/src/registry/mod.rs index 72649762..81f1dd03 100644 --- a/src/registry/mod.rs +++ b/src/registry/mod.rs @@ -8,6 +8,7 @@ use std::str::FromStr; use clap::{Args, Subcommand}; use crate::registry::generate::RegistryGenerateArgs; +use crate::registry::json_schema::RegistryJsonSchemaArgs; use crate::registry::resolve::RegistryResolveArgs; use crate::registry::search::RegistrySearchArgs; use crate::registry::stats::RegistryStatsArgs; @@ -19,6 +20,7 @@ use weaver_common::Logger; mod check; mod generate; +mod json_schema; mod resolve; mod search; mod stats; @@ -73,6 +75,11 @@ pub enum RegistrySubCommand { Stats(RegistryStatsArgs), /// Update markdown files that contain markers indicating the templates used to update the specified sections. UpdateMarkdown(RegistryUpdateMarkdownArgs), + /// Generate the JSON Schema of the resolved registry documents consumed by the template generator and the policy engine. + /// + /// The produced JSON Schema can be used to generate documentation of the resolved registry format or to generate code in your language of choice if you need to interact with the resolved registry format for any reason. + #[clap(verbatim_doc_comment)] + JsonSchema(RegistryJsonSchemaArgs), } /// Path to a semantic convention registry. @@ -156,5 +163,9 @@ pub fn semconv_registry(log: impl Logger + Sync + Clone, command: &RegistryComma update_markdown::command(log.clone(), &cache, args), Some(args.diagnostic.clone()), ), + RegistrySubCommand::JsonSchema(args) => CmdResult::new( + json_schema::command(log.clone(), &cache, args), + Some(args.diagnostic.clone()), + ), } } diff --git a/src/registry/resolve.rs b/src/registry/resolve.rs index 2009533b..2713108d 100644 --- a/src/registry/resolve.rs +++ b/src/registry/resolve.rs @@ -17,7 +17,7 @@ use crate::registry::RegistryArgs; use crate::util::{ check_policies, load_semconv_specs, resolve_semconv_specs, semconv_registry_path_from, }; -use crate::DiagnosticArgs; +use crate::{DiagnosticArgs, ExitDirectives}; /// Parameters for the `registry resolve` sub-command #[derive(Debug, Args)] @@ -65,7 +65,10 @@ pub(crate) fn command( logger: impl Logger + Sync + Clone, cache: &Cache, args: &RegistryResolveArgs, -) -> Result<(), DiagnosticMessages> { +) -> Result { + if args.output.is_none() { + logger.mute(); + } logger.loading(&format!("Resolving registry `{}`", args.registry.registry)); let registry_id = "default"; @@ -116,7 +119,10 @@ pub(crate) fn command( panic!("{}", e); }); - Ok(()) + Ok(ExitDirectives { + exit_code: 0, + quiet_mode: args.output.is_none(), + }) } #[cfg(test)] @@ -153,9 +159,9 @@ mod tests { })), }; - let exit_code = run_command(&cli, logger.clone()); + let exit_directive = run_command(&cli, logger.clone()); // The command should succeed. - assert_eq!(exit_code, 0); + assert_eq!(exit_directive.exit_code, 0); // Now, let's run the command again with the policy checks enabled. let cli = Cli { @@ -179,8 +185,8 @@ mod tests { })), }; - let exit_code = run_command(&cli, logger); + let exit_directive = run_command(&cli, logger); // The command should exit with an error code. - assert_eq!(exit_code, 1); + assert_eq!(exit_directive.exit_code, 1); } } diff --git a/src/registry/stats.rs b/src/registry/stats.rs index cbc8a3a4..bd1c1564 100644 --- a/src/registry/stats.rs +++ b/src/registry/stats.rs @@ -4,7 +4,7 @@ use crate::registry::RegistryArgs; use crate::util::{load_semconv_specs, resolve_semconv_specs, semconv_registry_path_from}; -use crate::DiagnosticArgs; +use crate::{DiagnosticArgs, ExitDirectives}; use clap::Args; use weaver_cache::Cache; use weaver_common::diagnostic::DiagnosticMessages; @@ -32,7 +32,7 @@ pub(crate) fn command( logger: impl Logger + Sync + Clone, cache: &Cache, args: &RegistryStatsArgs, -) -> Result<(), DiagnosticMessages> { +) -> Result { logger.loading(&format!( "Compute statistics on the registry `{}`", args.registry.registry @@ -52,7 +52,10 @@ pub(crate) fn command( let resolved_schema = resolve_semconv_specs(&mut registry, logger)?; display_schema_stats(&resolved_schema); - Ok(()) + Ok(ExitDirectives { + exit_code: 0, + quiet_mode: false, + }) } #[cfg(not(tarpaulin_include))] diff --git a/src/registry/update_markdown.rs b/src/registry/update_markdown.rs index b919bea6..7c07aa05 100644 --- a/src/registry/update_markdown.rs +++ b/src/registry/update_markdown.rs @@ -5,7 +5,7 @@ use crate::registry::RegistryArgs; use crate::util::semconv_registry_path_from; -use crate::DiagnosticArgs; +use crate::{DiagnosticArgs, ExitDirectives}; use clap::Args; use weaver_cache::Cache; use weaver_common::diagnostic::DiagnosticMessages; @@ -56,7 +56,7 @@ pub(crate) fn command( log: impl Logger + Sync + Clone, cache: &Cache, args: &RegistryUpdateMarkdownArgs, -) -> Result<(), DiagnosticMessages> { +) -> Result { fn is_markdown(entry: &walkdir::DirEntry) -> bool { let path = entry.path(); let extension = path.extension().unwrap_or_else(|| std::ffi::OsStr::new("")); @@ -108,7 +108,10 @@ pub(crate) fn command( panic!("weaver registry update-markdown failed."); } - Ok(()) + Ok(ExitDirectives { + exit_code: 0, + quiet_mode: false, + }) } #[cfg(test)] @@ -142,8 +145,8 @@ mod tests { })), }; - let exit_code = run_command(&cli, logger.clone()); + let exit_directive = run_command(&cli, logger.clone()); // The command should succeed. - assert_eq!(exit_code, 0); + assert_eq!(exit_directive.exit_code, 0); } } diff --git a/src/schema/resolve.rs b/src/schema/resolve.rs index 9e2a38f3..55866345 100644 --- a/src/schema/resolve.rs +++ b/src/schema/resolve.rs @@ -7,7 +7,7 @@ use crate::registry::RegistryArgs; use crate::util::{ check_policies, load_semconv_specs, resolve_semconv_specs, semconv_registry_path_from, }; -use crate::{format, DiagnosticArgs}; +use crate::{format, DiagnosticArgs, ExitDirectives}; use clap::Args; use std::path::PathBuf; use weaver_cache::Cache; @@ -56,7 +56,7 @@ pub(crate) fn command( logger: impl Logger + Sync + Clone, cache: &Cache, args: &SchemaResolveArgs, -) -> Result<(), DiagnosticMessages> { +) -> Result { logger.loading(&format!("Resolving schema `{}`", args.registry.registry)); let registry_id = "default"; @@ -99,7 +99,10 @@ pub(crate) fn command( panic!("{}", e); }); - Ok(()) + Ok(ExitDirectives { + exit_code: 0, + quiet_mode: args.output.is_none(), + }) } #[cfg(test)] @@ -136,9 +139,9 @@ mod tests { })), }; - let exit_code = run_command(&cli, logger.clone()); + let exit_directive = run_command(&cli, logger.clone()); // The command should succeed. - assert_eq!(exit_code, 0); + assert_eq!(exit_directive.exit_code, 0); // Now, let's run the command again with the policy checks enabled. let cli = Cli { @@ -161,8 +164,8 @@ mod tests { })), }; - let exit_code = run_command(&cli, logger); + let exit_directive = run_command(&cli, logger); // The command should exit with an error code. - assert_eq!(exit_code, 1); + assert_eq!(exit_directive.exit_code, 1); } } From 7182338b2ad664de4209384186ec462b84e9691b Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Thu, 30 May 2024 14:44:33 -0700 Subject: [PATCH 6/9] feat(json-schema): Add command to output the JSON schema of a ResolvedTelemetrySchema object. --- src/schema/json_schema.rs | 136 ++++++++++++++++++++++++++++++++++++++ src/schema/mod.rs | 11 +++ 2 files changed, 147 insertions(+) create mode 100644 src/schema/json_schema.rs diff --git a/src/schema/json_schema.rs b/src/schema/json_schema.rs new file mode 100644 index 00000000..20e5ce47 --- /dev/null +++ b/src/schema/json_schema.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Generate the JSON Schema of the resolved telemetry schema. + +use crate::{DiagnosticArgs, ExitDirectives}; +use clap::Args; +use miette::Diagnostic; +use schemars::schema_for; +use serde::Serialize; +use serde_json::to_string_pretty; +use std::path::PathBuf; +use weaver_cache::Cache; +use weaver_common::diagnostic::{DiagnosticMessage, DiagnosticMessages}; +use weaver_common::Logger; +use weaver_resolved_schema::ResolvedTelemetrySchema; + +/// Parameters for the `schema json-schema` sub-command +#[derive(Debug, Args)] +pub struct SchemaJsonSchemaArgs { + /// Output file to write the JSON schema to + /// If not specified, the JSON schema is printed to stdout + #[arg(short, long)] + output: Option, + + /// Parameters to specify the diagnostic format. + #[command(flatten)] + pub diagnostic: DiagnosticArgs, +} + +/// An error that can occur while generating a JSON Schema. +#[derive(thiserror::Error, Debug, Clone, PartialEq, Serialize, Diagnostic)] +#[non_exhaustive] +pub enum Error { + /// The serialization of the JSON schema failed. + #[error("The serialization of the JSON schema failed. Error: {error}")] + SerializationError { + /// The error that occurred. + error: String, + }, + + /// Writing to the file failed. + #[error("Writing to the file ‘{file}’ failed for the following reason: {error}")] + WriteError { + /// The path to the output file. + file: PathBuf, + /// The error that occurred. + error: String, + }, +} + +impl From for DiagnosticMessages { + fn from(error: Error) -> Self { + DiagnosticMessages::new(vec![DiagnosticMessage::new(error)]) + } +} + +/// Generate the JSON Schema of a Telemetry Schema and write the JSON schema to a +/// file or print it to stdout. +#[cfg(not(tarpaulin_include))] +pub(crate) fn command( + logger: impl Logger + Sync + Clone, + _cache: &Cache, + args: &SchemaJsonSchemaArgs, +) -> Result { + let json_schema = schema_for!(ResolvedTelemetrySchema); + + let json_schema_str = + to_string_pretty(&json_schema).map_err(|e| Error::SerializationError { + error: e.to_string(), + })?; + + if let Some(output) = &args.output { + logger.loading(&format!("Writing JSON schema to `{}`", output.display())); + std::fs::write(output, json_schema_str).map_err(|e| Error::WriteError { + file: output.clone(), + error: e.to_string(), + })?; + } else { + logger.log(&json_schema_str); + } + + Ok(ExitDirectives { + exit_code: 0, + quiet_mode: args.output.is_none(), + }) +} + +#[cfg(test)] +mod tests { + use weaver_common::in_memory; + use weaver_common::in_memory::LogMessage; + + use crate::cli::{Cli, Commands}; + use crate::run_command; + use crate::schema::json_schema::SchemaJsonSchemaArgs; + use crate::schema::{SchemaCommand, SchemaSubCommand}; + + #[test] + fn test_registry_json_schema() { + let logger = in_memory::Logger::new(0); + let cli = Cli { + debug: 0, + quiet: false, + command: Some(Commands::Schema(SchemaCommand { + command: SchemaSubCommand::JsonSchema(SchemaJsonSchemaArgs { + output: None, + diagnostic: Default::default(), + }), + })), + }; + + let exit_directive = run_command(&cli, logger.clone()); + // The command should succeed. + assert_eq!(exit_directive.exit_code, 0); + + // We should have a single log message with the JSON schema. + let messages = logger.messages(); + assert_eq!(messages.len(), 1); + + let message = &messages[0]; + if let LogMessage::Log(log) = message { + let value = + serde_json::from_str::(log).expect("Failed to parse JSON"); + let definitions = value + .as_object() + .expect("Expected a JSON object") + .get("definitions"); + assert!( + definitions.is_some(), + "Expected a 'definitions' key in the JSON schema" + ); + } else { + panic!("Expected a log message, but got: {:?}", message); + } + } +} diff --git a/src/schema/mod.rs b/src/schema/mod.rs index fd114184..74fef02e 100644 --- a/src/schema/mod.rs +++ b/src/schema/mod.rs @@ -2,8 +2,10 @@ //! Commands to manage a telemetry schema. +mod json_schema; mod resolve; +use crate::schema::json_schema::SchemaJsonSchemaArgs; use crate::schema::resolve::SchemaResolveArgs; use crate::CmdResult; use clap::{Args, Subcommand}; @@ -31,6 +33,11 @@ pub enum SchemaSubCommand { /// The process exits with a code of 0 if the resolution is successful. #[clap(verbatim_doc_comment)] Resolve(SchemaResolveArgs), + /// Generate the JSON Schema of a resolved telemetry schema. + /// + /// The produced JSON Schema can be used to generate documentation of the resolved registry format or to generate code in your language of choice if you need to interact with the resolved telemetry schema format for any reason. + #[clap(verbatim_doc_comment)] + JsonSchema(SchemaJsonSchemaArgs), } /// Manage a telemetry schema and return the exit code. @@ -45,5 +52,9 @@ pub fn telemetry_schema(log: impl Logger + Sync + Clone, command: &SchemaCommand resolve::command(log.clone(), &cache, args), Some(args.diagnostic.clone()), ), + SchemaSubCommand::JsonSchema(args) => CmdResult::new( + json_schema::command(log.clone(), &cache, args), + Some(args.diagnostic.clone()), + ), } } From d14e711c0028d10c21b260cbf287db549b6dc8d4 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Fri, 31 May 2024 14:50:15 -0700 Subject: [PATCH 7/9] chore(common): Replace Mutex by AtomicBool --- crates/weaver_common/src/lib.rs | 52 ++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/crates/weaver_common/src/lib.rs b/crates/weaver_common/src/lib.rs index b1a830e4..f2bf73f7 100644 --- a/crates/weaver_common/src/lib.rs +++ b/crates/weaver_common/src/lib.rs @@ -7,7 +7,7 @@ pub mod error; pub mod in_memory; pub mod quiet; -use std::sync::atomic::AtomicUsize; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; /// A trait that defines the interface of a logger. @@ -58,7 +58,15 @@ pub trait Logger { pub struct ConsoleLogger { logger: Arc>>, debug_level: u8, - mute: Arc>, + /// Mute all the messages except for the warnings and errors. + /// This flag is used to dynamically mute the logger. + /// + /// Ordering logic: + /// - Ordering::Acquire in load: Ensures that when a thread reads the muted flag, it sees all + /// preceding writes to that flag by other threads. + /// - Ordering::Release in store: Ensures that when a thread sets the muted flag, the store + /// operation is visible to other threads that subsequently perform an acquire load. + mute: Arc, } impl ConsoleLogger { @@ -68,7 +76,7 @@ impl ConsoleLogger { ConsoleLogger { logger: Arc::new(Mutex::new(paris::Logger::new())), debug_level, - mute: Arc::new(Mutex::new(false)), + mute: Arc::new(AtomicBool::new(false)), } } } @@ -76,8 +84,7 @@ impl ConsoleLogger { impl Logger for ConsoleLogger { /// Logs an trace message (only with debug enabled). fn trace(&self, message: &str) { - let mute = *self.mute.lock().expect("Failed to lock mute"); - if self.debug_level > 0 && !mute { + if self.debug_level > 0 && !self.mute.load(Ordering::Acquire) { _ = self .logger .lock() @@ -88,7 +95,7 @@ impl Logger for ConsoleLogger { /// Logs an info message. fn info(&self, message: &str) { - if *self.mute.lock().expect("Failed to lock mute") { + if self.mute.load(Ordering::Acquire) { return; } @@ -119,7 +126,7 @@ impl Logger for ConsoleLogger { /// Logs a success message. fn success(&self, message: &str) { - if *self.mute.lock().expect("Failed to lock mute") { + if self.mute.load(Ordering::Acquire) { return; } @@ -132,7 +139,7 @@ impl Logger for ConsoleLogger { /// Logs a newline. fn newline(&self, count: usize) { - if *self.mute.lock().expect("Failed to lock mute") { + if self.mute.load(Ordering::Acquire) { return; } @@ -145,7 +152,7 @@ impl Logger for ConsoleLogger { /// Indents the logger. fn indent(&self, count: usize) { - if *self.mute.lock().expect("Failed to lock mute") { + if self.mute.load(Ordering::Acquire) { return; } @@ -158,7 +165,7 @@ impl Logger for ConsoleLogger { /// Stops a loading message. fn done(&self) { - if *self.mute.lock().expect("Failed to lock mute") { + if self.mute.load(Ordering::Acquire) { return; } @@ -167,7 +174,7 @@ impl Logger for ConsoleLogger { /// Adds a style to the logger. fn add_style(&self, name: &str, styles: Vec<&'static str>) -> &Self { - if *self.mute.lock().expect("Failed to lock mute") { + if self.mute.load(Ordering::Acquire) { return self; } @@ -181,7 +188,7 @@ impl Logger for ConsoleLogger { /// Logs a loading message with a spinner. fn loading(&self, message: &str) { - if *self.mute.lock().expect("Failed to lock mute") { + if self.mute.load(Ordering::Acquire) { return; } @@ -194,7 +201,7 @@ impl Logger for ConsoleLogger { /// Forces the logger to not print a newline for the next message. fn same(&self) -> &Self { - if *self.mute.lock().expect("Failed to lock mute") { + if self.mute.load(Ordering::Acquire) { return self; } @@ -204,7 +211,7 @@ impl Logger for ConsoleLogger { /// Logs a message without icon. fn log(&self, message: &str) { - if *self.mute.lock().expect("Failed to lock mute") { + if self.mute.load(Ordering::Acquire) { return; } @@ -217,7 +224,10 @@ impl Logger for ConsoleLogger { /// Mute all the messages except for the warnings and errors. fn mute(&self) { - *self.mute.lock().expect("Failed to lock mute") = true; + // Ordering::Release: + // Ensures that when a thread sets the muted flag, the store operation is visible to other + // threads that subsequently perform an acquire load. + self.mute.store(true, Ordering::Release); } } @@ -301,13 +311,13 @@ impl TestLogger { /// Returns the number of warning messages logged. #[must_use] pub fn warn_count(&self) -> usize { - self.warn_count.load(std::sync::atomic::Ordering::Relaxed) + self.warn_count.load(Ordering::Relaxed) } /// Returns the number of error messages logged. #[must_use] pub fn error_count(&self) -> usize { - self.error_count.load(std::sync::atomic::Ordering::Relaxed) + self.error_count.load(Ordering::Relaxed) } } @@ -331,9 +341,7 @@ impl Logger for TestLogger { .lock() .expect("Failed to lock logger") .warn(message); - _ = self - .warn_count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + _ = self.warn_count.fetch_add(1, Ordering::Relaxed); } /// Logs an error message. @@ -343,9 +351,7 @@ impl Logger for TestLogger { .lock() .expect("Failed to lock logger") .error(message); - _ = self - .error_count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + _ = self.error_count.fetch_add(1, Ordering::Relaxed); } /// Logs a success message. From 3043bc0f6c953ddbfa76d284b2fc5aec93d4b36a Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Fri, 31 May 2024 15:26:03 -0700 Subject: [PATCH 8/9] chore(cli): Display the usage if no command is provided. --- src/cli.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/cli.rs b/src/cli.rs index 8bf21556..05da6218 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -9,7 +9,14 @@ use clap::{Parser, Subcommand}; /// Command line arguments. #[derive(Parser)] -#[command(author, version, about, long_about = None)] +#[command( + author, + version, + about, + long_about = None, + subcommand_required = true, + arg_required_else_help = true +)] pub struct Cli { /// Turn debugging information on #[arg(short, long, action = clap::ArgAction::Count)] From 9df9a42bb789580f896f538fe9ac2c9f2509550e Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Fri, 31 May 2024 15:32:30 -0700 Subject: [PATCH 9/9] chore: Remove schema command --- src/cli.rs | 3 - src/main.rs | 3 - src/schema/json_schema.rs | 136 ------------------------------ src/schema/mod.rs | 60 ------------- src/schema/resolve.rs | 171 -------------------------------------- 5 files changed, 373 deletions(-) delete mode 100644 src/schema/json_schema.rs delete mode 100644 src/schema/mod.rs delete mode 100644 src/schema/resolve.rs diff --git a/src/cli.rs b/src/cli.rs index 05da6218..ad7357f2 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -4,7 +4,6 @@ use crate::diagnostic::DiagnosticCommand; use crate::registry::RegistryCommand; -use crate::schema::SchemaCommand; use clap::{Parser, Subcommand}; /// Command line arguments. @@ -36,8 +35,6 @@ pub struct Cli { pub enum Commands { /// Manage Semantic Convention Registry Registry(RegistryCommand), - /// Manage Telemetry Schema - Schema(SchemaCommand), /// Manage Diagnostic Messages Diagnostic(DiagnosticCommand), } diff --git a/src/main.rs b/src/main.rs index e7cbfe94..449ebe5e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,13 +15,11 @@ use weaver_forge::{OutputDirective, TemplateEngine}; use crate::cli::{Cli, Commands}; use crate::diagnostic::DEFAULT_DIAGNOSTIC_TEMPLATES; -use crate::schema::telemetry_schema; mod cli; mod diagnostic; mod format; mod registry; -mod schema; mod util; /// Set of parameters used to specify the diagnostic format. @@ -103,7 +101,6 @@ fn main() { fn run_command(cli: &Cli, log: impl Logger + Sync + Clone) -> ExitDirectives { let cmd_result = match &cli.command { Some(Commands::Registry(params)) => semconv_registry(log.clone(), params), - Some(Commands::Schema(params)) => telemetry_schema(log.clone(), params), Some(Commands::Diagnostic(params)) => diagnostic::diagnostic(log.clone(), params), None => { return ExitDirectives { diff --git a/src/schema/json_schema.rs b/src/schema/json_schema.rs deleted file mode 100644 index 20e5ce47..00000000 --- a/src/schema/json_schema.rs +++ /dev/null @@ -1,136 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Generate the JSON Schema of the resolved telemetry schema. - -use crate::{DiagnosticArgs, ExitDirectives}; -use clap::Args; -use miette::Diagnostic; -use schemars::schema_for; -use serde::Serialize; -use serde_json::to_string_pretty; -use std::path::PathBuf; -use weaver_cache::Cache; -use weaver_common::diagnostic::{DiagnosticMessage, DiagnosticMessages}; -use weaver_common::Logger; -use weaver_resolved_schema::ResolvedTelemetrySchema; - -/// Parameters for the `schema json-schema` sub-command -#[derive(Debug, Args)] -pub struct SchemaJsonSchemaArgs { - /// Output file to write the JSON schema to - /// If not specified, the JSON schema is printed to stdout - #[arg(short, long)] - output: Option, - - /// Parameters to specify the diagnostic format. - #[command(flatten)] - pub diagnostic: DiagnosticArgs, -} - -/// An error that can occur while generating a JSON Schema. -#[derive(thiserror::Error, Debug, Clone, PartialEq, Serialize, Diagnostic)] -#[non_exhaustive] -pub enum Error { - /// The serialization of the JSON schema failed. - #[error("The serialization of the JSON schema failed. Error: {error}")] - SerializationError { - /// The error that occurred. - error: String, - }, - - /// Writing to the file failed. - #[error("Writing to the file ‘{file}’ failed for the following reason: {error}")] - WriteError { - /// The path to the output file. - file: PathBuf, - /// The error that occurred. - error: String, - }, -} - -impl From for DiagnosticMessages { - fn from(error: Error) -> Self { - DiagnosticMessages::new(vec![DiagnosticMessage::new(error)]) - } -} - -/// Generate the JSON Schema of a Telemetry Schema and write the JSON schema to a -/// file or print it to stdout. -#[cfg(not(tarpaulin_include))] -pub(crate) fn command( - logger: impl Logger + Sync + Clone, - _cache: &Cache, - args: &SchemaJsonSchemaArgs, -) -> Result { - let json_schema = schema_for!(ResolvedTelemetrySchema); - - let json_schema_str = - to_string_pretty(&json_schema).map_err(|e| Error::SerializationError { - error: e.to_string(), - })?; - - if let Some(output) = &args.output { - logger.loading(&format!("Writing JSON schema to `{}`", output.display())); - std::fs::write(output, json_schema_str).map_err(|e| Error::WriteError { - file: output.clone(), - error: e.to_string(), - })?; - } else { - logger.log(&json_schema_str); - } - - Ok(ExitDirectives { - exit_code: 0, - quiet_mode: args.output.is_none(), - }) -} - -#[cfg(test)] -mod tests { - use weaver_common::in_memory; - use weaver_common::in_memory::LogMessage; - - use crate::cli::{Cli, Commands}; - use crate::run_command; - use crate::schema::json_schema::SchemaJsonSchemaArgs; - use crate::schema::{SchemaCommand, SchemaSubCommand}; - - #[test] - fn test_registry_json_schema() { - let logger = in_memory::Logger::new(0); - let cli = Cli { - debug: 0, - quiet: false, - command: Some(Commands::Schema(SchemaCommand { - command: SchemaSubCommand::JsonSchema(SchemaJsonSchemaArgs { - output: None, - diagnostic: Default::default(), - }), - })), - }; - - let exit_directive = run_command(&cli, logger.clone()); - // The command should succeed. - assert_eq!(exit_directive.exit_code, 0); - - // We should have a single log message with the JSON schema. - let messages = logger.messages(); - assert_eq!(messages.len(), 1); - - let message = &messages[0]; - if let LogMessage::Log(log) = message { - let value = - serde_json::from_str::(log).expect("Failed to parse JSON"); - let definitions = value - .as_object() - .expect("Expected a JSON object") - .get("definitions"); - assert!( - definitions.is_some(), - "Expected a 'definitions' key in the JSON schema" - ); - } else { - panic!("Expected a log message, but got: {:?}", message); - } - } -} diff --git a/src/schema/mod.rs b/src/schema/mod.rs deleted file mode 100644 index 74fef02e..00000000 --- a/src/schema/mod.rs +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Commands to manage a telemetry schema. - -mod json_schema; -mod resolve; - -use crate::schema::json_schema::SchemaJsonSchemaArgs; -use crate::schema::resolve::SchemaResolveArgs; -use crate::CmdResult; -use clap::{Args, Subcommand}; -use weaver_cache::Cache; -use weaver_common::Logger; - -/// Parameters for the `registry` command -#[derive(Debug, Args)] -pub struct SchemaCommand { - /// Define the sub-commands for the `schema` command - #[clap(subcommand)] - pub command: SchemaSubCommand, -} - -/// Sub-commands to manage a `schema`. -#[derive(Debug, Subcommand)] -#[clap(verbatim_doc_comment)] -pub enum SchemaSubCommand { - /// Resolves a telemetry schema. - /// - /// Rego policies present in the registry or specified using -p or --policy will be automatically validated by the policy engine before the artifact generation phase. - /// - /// Note: The `-d` and `--registry-git-sub-dir` options are only used when the registry is a Git URL otherwise these options are ignored. - /// - /// The process exits with a code of 0 if the resolution is successful. - #[clap(verbatim_doc_comment)] - Resolve(SchemaResolveArgs), - /// Generate the JSON Schema of a resolved telemetry schema. - /// - /// The produced JSON Schema can be used to generate documentation of the resolved registry format or to generate code in your language of choice if you need to interact with the resolved telemetry schema format for any reason. - #[clap(verbatim_doc_comment)] - JsonSchema(SchemaJsonSchemaArgs), -} - -/// Manage a telemetry schema and return the exit code. -pub fn telemetry_schema(log: impl Logger + Sync + Clone, command: &SchemaCommand) -> CmdResult { - let cache = match Cache::try_new() { - Ok(cache) => cache, - Err(e) => return CmdResult::new(Err(e.into()), None), - }; - - match &command.command { - SchemaSubCommand::Resolve(args) => CmdResult::new( - resolve::command(log.clone(), &cache, args), - Some(args.diagnostic.clone()), - ), - SchemaSubCommand::JsonSchema(args) => CmdResult::new( - json_schema::command(log.clone(), &cache, args), - Some(args.diagnostic.clone()), - ), - } -} diff --git a/src/schema/resolve.rs b/src/schema/resolve.rs deleted file mode 100644 index 55866345..00000000 --- a/src/schema/resolve.rs +++ /dev/null @@ -1,171 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Resolve a telemetry schema. - -use crate::format::Format; -use crate::registry::RegistryArgs; -use crate::util::{ - check_policies, load_semconv_specs, resolve_semconv_specs, semconv_registry_path_from, -}; -use crate::{format, DiagnosticArgs, ExitDirectives}; -use clap::Args; -use std::path::PathBuf; -use weaver_cache::Cache; -use weaver_common::diagnostic::DiagnosticMessages; -use weaver_common::Logger; -use weaver_semconv::registry::SemConvRegistry; - -/// Parameters for the `schema resolve` sub-command -#[derive(Debug, Args)] -pub struct SchemaResolveArgs { - /// Parameters to specify the semantic convention registry - #[command(flatten)] - registry: RegistryArgs, - - /// Output file to write the resolved schema to - /// If not specified, the resolved schema is printed to stdout - #[arg(short, long)] - output: Option, - - /// Output format for the resolved schema - /// If not specified, the resolved schema is printed in YAML format - /// Supported formats: yaml, json - /// Default format: yaml - /// Example: `--format json` - #[arg(short, long, default_value = "yaml")] - format: Format, - - /// Optional list of policy files to check against the files of the semantic - /// convention registry. - #[arg(short = 'p', long = "policy")] - pub policies: Vec, - - /// Skip the policy checks. - #[arg(long, default_value = "false")] - pub skip_policies: bool, - - /// Parameters to specify the diagnostic format. - #[command(flatten)] - pub diagnostic: DiagnosticArgs, -} - -/// Resolve a telemetry schema and write the resolved schema to a -/// file or print it to stdout. -#[cfg(not(tarpaulin_include))] -pub(crate) fn command( - logger: impl Logger + Sync + Clone, - cache: &Cache, - args: &SchemaResolveArgs, -) -> Result { - logger.loading(&format!("Resolving schema `{}`", args.registry.registry)); - - let registry_id = "default"; - let registry_path = - semconv_registry_path_from(&args.registry.registry, &args.registry.registry_git_sub_dir); - - // Load the semantic convention registry into a local cache. - let semconv_specs = load_semconv_specs(®istry_path, cache, logger.clone())?; - - if !args.skip_policies { - check_policies( - ®istry_path, - cache, - &args.policies, - &semconv_specs, - logger.clone(), - )?; - } - - let mut registry = SemConvRegistry::from_semconv_specs(registry_id, semconv_specs); - let schema = resolve_semconv_specs(&mut registry, logger.clone())?; - - // Serialize the resolved schema and write it - // to a file or print it to stdout. - format::apply_format(&args.format, &schema) - .map_err(|e| format!("Failed to serialize the registry: {e:?}")) - .and_then(|s| { - if let Some(ref path) = args.output { - // Write the resolved registry to a file. - std::fs::write(path, s) - .map_err(|e| format!("Failed to write the resolved registry to file: {e:?}")) - } else { - // Print the resolved registry to stdout. - println!("{}", s); - Ok(()) - } - }) - .unwrap_or_else(|e| { - // Capture all the errors - panic!("{}", e); - }); - - Ok(ExitDirectives { - exit_code: 0, - quiet_mode: args.output.is_none(), - }) -} - -#[cfg(test)] -mod tests { - use weaver_common::TestLogger; - - use crate::cli::{Cli, Commands}; - use crate::format::Format; - use crate::registry::{RegistryArgs, RegistryPath}; - use crate::run_command; - use crate::schema::resolve::SchemaResolveArgs; - use crate::schema::{SchemaCommand, SchemaSubCommand}; - - #[test] - fn test_schema_resolve() { - let logger = TestLogger::new(); - let cli = Cli { - debug: 0, - quiet: false, - command: Some(Commands::Schema(SchemaCommand { - command: SchemaSubCommand::Resolve(SchemaResolveArgs { - registry: RegistryArgs { - registry: RegistryPath::Local( - "crates/weaver_codegen_test/semconv_registry/".to_owned(), - ), - registry_git_sub_dir: None, - }, - output: None, - format: Format::Yaml, - policies: vec![], - skip_policies: true, - diagnostic: Default::default(), - }), - })), - }; - - let exit_directive = run_command(&cli, logger.clone()); - // The command should succeed. - assert_eq!(exit_directive.exit_code, 0); - - // Now, let's run the command again with the policy checks enabled. - let cli = Cli { - debug: 0, - quiet: false, - command: Some(Commands::Schema(SchemaCommand { - command: SchemaSubCommand::Resolve(SchemaResolveArgs { - registry: RegistryArgs { - registry: RegistryPath::Local( - "crates/weaver_codegen_test/semconv_registry/".to_owned(), - ), - registry_git_sub_dir: None, - }, - output: None, - format: Format::Json, - policies: vec![], - skip_policies: false, - diagnostic: Default::default(), - }), - })), - }; - - let exit_directive = run_command(&cli, logger); - // The command should exit with an error code. - assert_eq!(exit_directive.exit_code, 1); - } -}