Add driver tests and figure out why the interner isn't working

.github/workflows/test.yml

@@ -31,6 +31,7 @@ jobs:
         path: |
           Cargo.lock
           target/
+          !target/*/cobalt-*
     - name: Run Clippy
       run: cargo clippy
       env:

cobalt-ast/src/types/custom.rs

@@ -226,7 +226,7 @@ impl<'de> Deserialize<'de> for CustomHeader {
             where
                 A: MapAccess<'de>,
             {
-                while let Some(k) = map.next_key()? {
+                while let Some(k) = map.next_key::<Box<str>>()? {
                     let v = map.next_value_seed(CIProxySeed(self.0))?;
                     assert!(
                         CUSTOM_DATA

cobalt-ast/src/types/int.rs

@@ -1,5 +1,6 @@
 use super::*;
 use inkwell::IntPredicate::*;
+pub(crate) static INTERN: Interner<(u16, bool)> = Interner::new();
 #[derive(Debug, ConstIdentify, PartialEq, Eq, Hash, Display, RefCastCustom)]
 #[display(fmt = "{}{}", r#"if _0.1 {"u"} else {"i"}"#, "_0.0")]
 #[repr(transparent)]
@@ -8,7 +9,6 @@ impl Int {
     #[ref_cast_custom]
     fn from_ref(val: &(u16, bool)) -> &Self;
     pub fn new(bits: u16, unsigned: bool) -> &'static Self {
-        static INTERN: Interner<(u16, bool)> = Interner::new();
         Self::from_ref(INTERN.intern((bits, unsigned)))
     }
     pub fn signed(bits: u16) -> &'static Self {

cobalt-build/src/lib.rs

@@ -18,6 +18,8 @@ use std::io::{self, prelude::*, ErrorKind};
 use std::path::{Path, PathBuf};
 use thiserror::Error;
 
+pub const HOST_TRIPLE: &str = env!("HOST");
+
 #[derive(Debug, Clone, Copy, Error)]
 #[error("build failed because of {0} compiler error{}", if *.0 == 1 {""} else {"s"})]
 pub struct CompileErrors(pub usize);

cobalt-cli/src/lib.rs

@@ -19,6 +19,8 @@ use std::io::{prelude::*, BufReader};
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::time::{Duration, Instant};
+#[cfg(test)]
+mod tests;
 
 #[derive(Default, Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
 pub enum OutputType {

cobalt-cli/src/tests/hello_world.rs

@@ -0,0 +1,122 @@
+use super::*;
+
+#[test]
+fn test_hello_world_aot() {
+    let input_path_str = "src/tests/inputs/hello_world.co";
+    let output_path_str = "src/tests/outputs/hello_world";
+
+    let input = match clio::Input::new(input_path_str) {
+        Ok(p) => p,
+        Err(err) => panic!("(clio) failed to load input: {err}"),
+    };
+    let output = match clio::OutputPath::new(output_path_str) {
+        Ok(p) => p,
+        Err(err) => panic!("(clio) failed to load output: {err}"),
+    };
+
+    let cli = Cli::Aot {
+        input,
+        output: Some(output),
+        linked: vec![],
+        link_dirs: vec![],
+        headers: vec![],
+        triple: None,
+        emit: OutputType::Executable,
+        profile: None,
+        continue_if_err: true,
+        debug_mangle: false,
+        no_default_link: false,
+        timings: false,
+    };
+
+    if let Err(err) = driver(cli) {
+        panic!("failure while compiling file: {err}")
+    };
+
+    let command_output = Command::new(output_path_str)
+        .output()
+        .expect("Failed to execute command");
+    let output_str =
+        std::str::from_utf8(&command_output.stdout).expect("Output is not valid UTF-8");
+    assert_eq!(output_str.trim(), "Hello, world!");
+}
+
+#[test]
+fn test_hello_world_aot_linked() {
+    let input_main_path_str = "src/tests/inputs/hello_world_linked_main.co";
+    let input_lib_path_str = "src/tests/inputs/hello_world_linked_lib.co";
+    let output_main_path_str = "src/tests/outputs/hello_world_linked";
+    let output_lib_path_str = Path::new("src/tests/outputs").join(cobalt_build::libs::format_lib(
+        "hello_world_linked",
+        cobalt_build::HOST_TRIPLE,
+        true,
+    ));
+    // ---
+
+    let input_lib = match clio::Input::new(input_lib_path_str) {
+        Ok(p) => p,
+        Err(err) => panic!("(clio) failed to load input: {err}"),
+    };
+    let output_lib = match clio::OutputPath::new(&output_lib_path_str) {
+        Ok(p) => p,
+        Err(err) => panic!("(clio) failed to load output: {err}"),
+    };
+
+    let cli_lib = Cli::Aot {
+        input: input_lib,
+        output: Some(output_lib),
+        linked: vec![],
+        link_dirs: vec![],
+        headers: vec![],
+        triple: None,
+        emit: OutputType::Library,
+        profile: None,
+        continue_if_err: true,
+        debug_mangle: false,
+        no_default_link: false,
+        timings: false,
+    };
+
+    if let Err(err) = driver(cli_lib) {
+        panic!("failure while compiling file: {err}")
+    };
+
+    // ---
+
+    let input_main = match clio::Input::new(input_main_path_str) {
+        Ok(p) => p,
+        Err(err) => panic!("(clio) failed to load input: {err}"),
+    };
+    let output_main = match clio::OutputPath::new(output_main_path_str) {
+        Ok(p) => p,
+        Err(err) => panic!("(clio) failed to load output: {err}"),
+    };
+
+    let cli_main = Cli::Aot {
+        input: input_main,
+        output: Some(output_main),
+        linked: vec!["hello_world_linked".to_string()],
+        link_dirs: vec!["src/tests/outputs".into()],
+        headers: vec![],
+        triple: None,
+        emit: OutputType::Executable,
+        profile: None,
+        continue_if_err: true,
+        debug_mangle: false,
+        no_default_link: false,
+        timings: false,
+    };
+
+    if let Err(err) = driver(cli_main) {
+        panic!("failure while compiling file: {err}")
+    };
+
+    // ---
+
+    let command_output = Command::new(output_main_path_str)
+        .output()
+        .expect("Failed to execute command");
+    let output_str =
+        std::str::from_utf8(&command_output.stdout).expect("Output is not valid UTF-8");
+    assert_eq!(output_str.trim(), "Hello, world!");
+}

cobalt-cli/src/tests/inputs/hello_world.co

@@ -0,0 +1,6 @@
+@C(extern) fn puts(str: *u8);
+
+fn main(): i32 = {
+    puts("Hello, world!"c);
+    0
+};

cobalt-cli/src/tests/inputs/hello_world_linked_lib.co

@@ -0,0 +1,5 @@
+@C(extern) fn puts(str: *u8);
+
+@export fn hello_world() = {
+    puts("Hello, world!"c);
+};

cobalt-cli/src/tests/inputs/hello_world_linked_main.co

@@ -0,0 +1,4 @@
+fn main(): i32 = {
+    hello_world();
+    0
+};

cobalt-cli/src/tests/mod.rs

@@ -0,0 +1,3 @@
+use super::*;
+
+mod hello_world;

cobalt-cli/src/tests/outputs/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

cobalt-utils/Cargo.toml

@@ -9,6 +9,11 @@ description.workspace = true
 documentation.workspace = true
 
 [dependencies]
-aovec = "1.1.0"
-hashbrown = { version = "0.14.2", features = ["raw"] }
+boxcar = "0.2.4"
+bumpalo = { version = "3.14.0", features = ["boxed", "std"] }
+bumpalo-herd = "0.1.2"
+hashbrown = "0.14.2"
 once_cell = "1.18.0"
+ouroboros = "0.18.2"
+parking_lot = "0.12.1"
+thread_local = "1.1.7"

cobalt-utils/src/intern.rs

@@ -1,65 +1,132 @@
-use hashbrown::raw::*;
+use bumpalo_herd::*;
+use hashbrown::HashTable;
 use once_cell::sync::Lazy;
-use std::hash::{Hash, Hasher};
-use std::sync::RwLock;
-#[inline]
-fn hash(val: &impl Hash) -> u64 {
-    let mut state = std::collections::hash_map::DefaultHasher::default();
-    val.hash(&mut state);
-    state.finish()
+use parking_lot::RwLock;
+use std::borrow::*;
+use std::collections::hash_map::RandomState;
+use std::fmt;
+use std::hash::*;
+use thread_local::ThreadLocal;
+
+enum MaybeLazy<T> {
+    Eager(T),
+    Lazy(Lazy<T>),
+}
+impl<T> std::ops::Deref for MaybeLazy<T> {
+    type Target = T;
+    fn deref(&self) -> &T {
+        match self {
+            Self::Eager(val) => val,
+            Self::Lazy(val) => val,
+        }
+    }
+}
+
+#[ouroboros::self_referencing]
+struct InternerInner<T: Eq + Hash + Send + Sync + 'static> {
+    owner: Herd,
+    #[borrows(owner)]
+    #[not_covariant]
+    member: ThreadLocal<Member<'this>>,
+    #[borrows(owner)]
+    #[not_covariant]
+    set: RwLock<HashTable<&'this T>>,
 }
-pub struct Interner<'a, T: PartialEq + Eq + Hash> {
-    vec: Lazy<aovec::Aovec<T>>,
-    map: RwLock<RawTable<(&'a T, usize)>>,
+
+pub struct Interner<T: Eq + Hash + Send + Sync + 'static, S = RandomState>(
+    Lazy<InternerInner<T>>,
+    MaybeLazy<S>,
+);
+
+impl<T: fmt::Debug + Eq + Hash + Send + Sync + 'static, S> fmt::Debug for Interner<T, S> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.with_set(|set| fmt::Debug::fmt(set, f))
+    }
 }
-impl<'a, K: PartialEq + Eq + Hash> Interner<'a, K> {
+impl<T: Eq + Hash + Send + Sync + 'static, S> Interner<T, S> {
+    pub const fn with_hasher(state: S) -> Self {
+        Self(
+            Lazy::new(|| {
+                InternerInner::new(Herd::new(), |_| ThreadLocal::new(), |_| RwLock::default())
+            }),
+            MaybeLazy::Eager(state),
+        )
+    }
+}
+impl<T: Eq + Hash + Send + Sync + 'static, S: Default> Interner<T, S> {
     pub const fn new() -> Self {
-        Self {
-            vec: Lazy::new(|| aovec::Aovec::new(16)),
-            map: RwLock::new(RawTable::new()),
-        }
+        Self(
+            Lazy::new(|| {
+                InternerInner::new(Herd::new(), |_| ThreadLocal::new(), |_| RwLock::default())
+            }),
+            MaybeLazy::Lazy(Lazy::new(S::default)),
+        )
     }
-    pub fn intern(&'a self, key: K) -> &K {
-        let hashed = hash(&key);
-        let lock = self.map.read().unwrap();
-        if let Some(k) = lock.get(hashed, |v| v.0 == &key).map(|x| x.1) {
-            &self.vec[k]
-        } else {
-            std::mem::drop(lock);
-            let mut lock = self.map.write().unwrap();
-            let idx = self.vec.push(key);
-            let val = &self.vec[idx];
-            lock.insert(hashed, (val, idx), |v| hash(&v.0));
-            val
-        }
+}
+impl<T: Eq + Hash + Send + Sync + 'static, S: BuildHasher> Interner<T, S> {
+    pub fn intern<Q: Eq + Hash + Into<T>>(&self, key: Q) -> &T
+    where
+        T: Borrow<Q>,
+    {
+        let builder = &*self.1;
+        let hash = builder.hash_one(&key);
+        self.0.with_set(|set| {
+            let lock = set.read();
+            if let Some(r) = lock.find(hash, |m| <T as Borrow<Q>>::borrow(m) == &key) {
+                // safety: lifetime can't be invalidated as long as self is not moved
+                unsafe { crate::new_lifetime(r) }
+            } else {
+                std::mem::drop(lock);
+                let ret = self.0.with_member(|m| unsafe {
+                    crate::new_lifetime(
+                        m.get_or(|| crate::new_lifetime(self.0.borrow_owner()).get())
+                            .alloc(key.into()),
+                    )
+                });
+                set.write()
+                    .insert_unique(hash, ret, |v| builder.hash_one(v));
+                ret
+            }
+        })
     }
-    pub fn intern_ref<R: PartialEq + ?Sized, Q: Hash + Eq + AsRef<R> + ?Sized + 'a>(
-        &'a self,
-        key: &Q,
-    ) -> &K
+    pub fn intern_ref<'a, Q: Eq + Hash + ToOwned + ?Sized>(&'a self, key: &Q) -> &'a T
     where
-        K: std::borrow::Borrow<Q> + AsRef<R>,
-        for<'b> &'b Q: Into<K>,
+        T: Borrow<Q>,
+        Q::Owned: Into<T>,
     {
-        let hashed = hash(&key);
-        let lock = self.map.read().unwrap();
-        if let Some(k) = lock
-            .get(hashed, |v| key.as_ref() == v.0.as_ref())
-            .map(|x| x.1)
-        {
-            &self.vec[k]
-        } else {
-            std::mem::drop(lock);
-            let mut lock = self.map.write().unwrap();
-            let idx = self.vec.push(key.into());
-            let val = &self.vec[idx];
-            lock.insert(hashed, (val, idx), |v| hash(&v.0));
-            val
-        }
+        let builder = &*self.1;
+        let hash = builder.hash_one(key);
+        self.0.with_set(|set| {
+            let lock = set.read();
+            if let Some(r) = lock.find(hash, |m| <T as Borrow<Q>>::borrow(m) == key) {
+                // safety: lifetime can't be invalidated as long as self is not moved
+                unsafe { crate::new_lifetime(r) }
+            } else {
+                std::mem::drop(lock);
+                let ret = self.0.with_member(|m| unsafe {
+                    crate::new_lifetime(
+                        m.get_or(|| crate::new_lifetime(self.0.borrow_owner()).get())
+                            .alloc(key.to_owned().into()),
+                    )
+                });
+                set.write()
+                    .insert_unique(hash, ret, |v| builder.hash_one(v));
+                ret
+            }
+        })
+    }
+}
+impl<T: Eq + Hash + Send + Sync + 'static, S> Drop for Interner<T, S> {
+    fn drop(&mut self) {
+        self.0.with_set_mut(|set| {
+            set.get_mut()
+                .iter_mut()
+                .for_each(|p| unsafe { std::ptr::drop_in_place::<T>(*p as *const T as *mut T) })
+        })
     }
 }
 
-impl<T: Clone + PartialEq + Eq + Hash> Default for Interner<'_, T> {
+impl<T: Eq + Hash + Send + Sync + 'static, S: Default> Default for Interner<T, S> {
     fn default() -> Self {
         Self::new()
     }