Implement merging semver-compatible interfaces in imports

This commit is the first half and the meat of the implementation
of #1774 where, by default, WIT processing tools will now merge world
imports when possible based on semver versions. The precise shape of how
this is done has gone through a few iterations and this is the end
result I've ended up settling on. This should handle all the various
cases we're interested in and continue to produce valid worlds within a
`Resolve` (with the help of `elaborate_world` added in #1800). CLI
tooling has been updated with flags to configure this behavior but the
behavior is now enabled by default.

crates/wit-component/src/encoding.rs

@@ -80,8 +80,10 @@ use crate::validation::{
 use crate::StringEncoding;
 use anyhow::{anyhow, bail, Context, Result};
 use indexmap::{IndexMap, IndexSet};
+use std::borrow::Cow;
 use std::collections::HashMap;
 use std::hash::Hash;
+use std::mem;
 use wasm_encoder::*;
 use wasmparser::Validator;
 use wit_parser::{
@@ -1968,6 +1970,7 @@ pub struct ComponentEncoder {
     adapters: IndexMap<String, Adapter>,
     import_name_map: HashMap<String, String>,
     realloc_via_memory_grow: bool,
+    merge_imports_based_on_semver: Option<bool>,
 }
 
 impl ComponentEncoder {
@@ -1977,14 +1980,11 @@ impl ComponentEncoder {
     /// It will also add any producers information inside the component type information to the
     /// core module.
     pub fn module(mut self, module: &[u8]) -> Result<Self> {
-        let (wasm, metadata) = metadata::decode(module)?;
-        let wasm = wasm.as_deref().unwrap_or(module);
-        let world = self
-            .metadata
-            .merge(metadata)
+        let (wasm, metadata) = self.decode(module)?;
+        let exports = self
+            .merge_metadata(metadata)
             .context("failed merge WIT metadata for module with previous metadata")?;
-        self.main_module_exports
-            .extend(self.metadata.resolve.worlds[world].exports.keys().cloned());
+        self.main_module_exports.extend(exports);
         self.module = if let Some(producers) = &self.metadata.producers {
             producers.add_to_wasm(&wasm)?
         } else {
@@ -1993,12 +1993,38 @@ impl ComponentEncoder {
         Ok(self)
     }
 
+    fn decode<'a>(&self, wasm: &'a [u8]) -> Result<(Cow<'a, [u8]>, Bindgen)> {
+        let (bytes, metadata) =
+            metadata::decode(wasm, self.merge_imports_based_on_semver.unwrap_or(true))?;
+        match bytes {
+            Some(wasm) => Ok((Cow::Owned(wasm), metadata)),
+            None => Ok((Cow::Borrowed(wasm), metadata)),
+        }
+    }
+
+    fn merge_metadata(&mut self, metadata: Bindgen) -> Result<IndexSet<WorldKey>> {
+        self.metadata
+            .merge(metadata, self.merge_imports_based_on_semver.unwrap_or(true))
+    }
+
     /// Sets whether or not the encoder will validate its output.
     pub fn validate(mut self, validate: bool) -> Self {
         self.validate = validate;
         self
     }
 
+    /// Sets whether to merge imports based on semver to the specified value.
+    ///
+    /// This affects how when to WIT worlds are merged together, for example
+    /// from two different libraries, whether their imports are unified when the
+    /// semver version ranges for interface allow it.
+    ///
+    /// This is enabled by default.
+    pub fn merge_imports_based_on_semver(mut self, merge: bool) -> Self {
+        self.merge_imports_based_on_semver = Some(merge);
+        self
+    }
+
     /// Specifies a new adapter which is used to translate from a historical
     /// wasm ABI to the canonical ABI and the `interface` provided.
     ///
@@ -2042,36 +2068,18 @@ impl ComponentEncoder {
         bytes: &[u8],
         library_info: Option<LibraryInfo>,
     ) -> Result<Self> {
-        let (wasm, metadata) = metadata::decode(bytes)?;
-        let wasm = wasm.as_deref().unwrap_or(bytes);
+        let (wasm, mut metadata) = self.decode(bytes)?;
         // Merge the adapter's document into our own document to have one large
         // document, and then afterwards merge worlds as well.
         //
-        // The first `merge` operation will interleave equivalent packages from
-        // each adapter into packages that are stored within our own resolve.
-        // The second `merge_worlds` operation will then ensure that both the
-        // adapter and the main module have compatible worlds, meaning that they
-        // either import the same items or they import disjoint items, for
-        // example.
-        let world = self
-            .metadata
-            .resolve
-            .merge(metadata.resolve)
-            .with_context(|| {
-                format!("failed to merge WIT packages of adapter `{name}` into main packages")
-            })?
-            .map_world(metadata.world, None)?;
-        self.metadata
-            .resolve
-            .merge_worlds(world, self.metadata.world)
-            .with_context(|| {
-                format!("failed to merge WIT world of adapter `{name}` into main package")
-            })?;
-        let exports = self.metadata.resolve.worlds[world]
-            .exports
-            .keys()
-            .cloned()
-            .collect();
+        // Note that the `metadata` tracking import/export encodings is removed
+        // since this adapter can get different lowerings and is allowed to
+        // differ from the main module. This is then tracked within the
+        // `Adapter` structure produced below.
+        let adapter_metadata = mem::take(&mut metadata.metadata);
+        let exports = self.merge_metadata(metadata).with_context(|| {
+            format!("failed to merge WIT packages of adapter `{name}` into main packages")
+        })?;
         if let Some(library_info) = &library_info {
             // Validate that all referenced modules can be resolved.
             for (_, instance) in &library_info.arguments {
@@ -2100,7 +2108,7 @@ impl ComponentEncoder {
             name.to_string(),
             Adapter {
                 wasm: wasm.to_vec(),
-                metadata: metadata.metadata,
+                metadata: adapter_metadata,
                 required_exports: exports,
                 library_info,
             },

crates/wit-component/src/linking.rs

@@ -1222,6 +1222,11 @@ pub struct Linker {
     ///
     /// If `None`, use `DEFAULT_STACK_SIZE_BYTES`.
     stack_size: Option<u32>,
+
+    /// This affects how when to WIT worlds are merged together, for example
+    /// from two different libraries, whether their imports are unified when the
+    /// semver version ranges for interface allow it.
+    merge_imports_based_on_semver: Option<bool>,
 }
 
 impl Linker {
@@ -1269,6 +1274,16 @@ impl Linker {
         self
     }
 
+    /// This affects how when to WIT worlds are merged together, for example
+    /// from two different libraries, whether their imports are unified when the
+    /// semver version ranges for interface allow it.
+    ///
+    /// This is enabled by default.
+    pub fn merge_imports_based_on_semver(mut self, merge: bool) -> Self {
+        self.merge_imports_based_on_semver = Some(merge);
+        self
+    }
+
     /// Encode the component and return the bytes
     pub fn encode(mut self) -> Result<Vec<u8>> {
         if self.use_built_in_libdl {
@@ -1290,8 +1305,14 @@ impl Linker {
             .libraries
             .iter()
             .map(|(name, module, dl_openable)| {
-                Metadata::try_new(name, *dl_openable, module, &adapter_names)
-                    .with_context(|| format!("failed to extract linking metadata from {name}"))
+                Metadata::try_new(
+                    name,
+                    *dl_openable,
+                    module,
+                    &adapter_names,
+                    self.merge_imports_based_on_semver.unwrap_or(true),
+                )
+                .with_context(|| format!("failed to extract linking metadata from {name}"))
             })
             .collect::<Result<Vec<_>>>()?;
 

crates/wit-component/src/linking/metadata.rs

@@ -214,8 +214,9 @@ impl<'a> Metadata<'a> {
         dl_openable: bool,
         module: &'a [u8],
         adapter_names: &HashSet<&str>,
+        merge_imports_based_on_semver: bool,
     ) -> Result<Self> {
-        let bindgen = crate::metadata::decode(module)?.1;
+        let bindgen = crate::metadata::decode(module, merge_imports_based_on_semver)?.1;
         let has_component_exports = !bindgen.resolve.worlds[bindgen.world].exports.is_empty();
 
         let mut result = Self {

crates/wit-component/src/metadata.rs

@@ -44,14 +44,14 @@
 use crate::validation::BARE_FUNC_MODULE_NAME;
 use crate::{DecodedWasm, StringEncoding};
 use anyhow::{bail, Context, Result};
-use indexmap::IndexMap;
+use indexmap::{IndexMap, IndexSet};
 use std::borrow::Cow;
 use wasm_encoder::{
     ComponentBuilder, ComponentExportKind, ComponentType, ComponentTypeRef, CustomSection,
 };
 use wasm_metadata::Producers;
 use wasmparser::{BinaryReader, Encoding, Parser, Payload};
-use wit_parser::{Package, PackageName, Resolve, World, WorldId, WorldItem};
+use wit_parser::{Package, PackageName, Resolve, World, WorldId, WorldItem, WorldKey};
 
 const CURRENT_VERSION: u8 = 0x04;
 const CUSTOM_SECTION_NAME: &str = "wit-component-encoding";
@@ -133,7 +133,10 @@ pub struct ModuleMetadata {
 /// If a `component-type` custom section was found then a new binary is
 /// optionally returned with the custom sections stripped out. If no
 /// `component-type` custom sections are found then `None` is returned.
-pub fn decode(wasm: &[u8]) -> Result<(Option<Vec<u8>>, Bindgen)> {
+pub fn decode(
+    wasm: &[u8],
+    merge_imports_based_on_semver: bool,
+) -> Result<(Option<Vec<u8>>, Bindgen)> {
     let mut ret = Bindgen::default();
     let mut new_module = wasm_encoder::Module::new();
 
@@ -144,7 +147,7 @@ pub fn decode(wasm: &[u8]) -> Result<(Option<Vec<u8>>, Bindgen)> {
             wasmparser::Payload::CustomSection(cs) if cs.name().starts_with("component-type") => {
                 let data = Bindgen::decode_custom_section(cs.data())
                     .with_context(|| format!("decoding custom section {}", cs.name()))?;
-                ret.merge(data)
+                ret.merge(data, merge_imports_based_on_semver)
                     .with_context(|| format!("updating metadata for section {}", cs.name()))?;
                 found_custom = true;
             }
@@ -298,7 +301,14 @@ impl Bindgen {
     ///
     /// Note that at this time there's no support for changing string encodings
     /// between metadata.
-    pub fn merge(&mut self, other: Bindgen) -> Result<WorldId> {
+    ///
+    /// This function returns the set of exports that the main world of
+    /// `other` added to the world in `self`.
+    pub fn merge(
+        &mut self,
+        other: Bindgen,
+        merge_imports_based_on_semver: bool,
+    ) -> Result<IndexSet<WorldKey>> {
         let Bindgen {
             resolve,
             world,
@@ -315,10 +325,18 @@ impl Bindgen {
             .merge(resolve)
             .context("failed to merge WIT package sets together")?
             .map_world(world, None)?;
+        let exports = self.resolve.worlds[world].exports.keys().cloned().collect();
         self.resolve
             .merge_worlds(world, self.world)
             .context("failed to merge worlds from two documents")?;
 
+        // If requested additionally deduplicate any imports based on semver.
+        if merge_imports_based_on_semver {
+            self.resolve
+                .merge_world_imports_based_on_semver(self.world)
+                .context("failed to deduplicate imports based on semver")?;
+        }
+
         for (name, encoding) in export_encodings {
             let prev = self
                 .metadata
@@ -349,7 +367,7 @@ impl Bindgen {
             }
         }
 
-        Ok(world)
+        Ok(exports)
     }
 }