feat: Add test(should_fail) attribute for tests that are meant to fail

crates/lsp/src/lib.rs

@@ -199,8 +199,8 @@ fn on_code_lens_request(
         let tests = context
             .get_all_test_functions_in_crate_matching(&crate_id, FunctionNameMatch::Anything);
 
-        for (func_name, func_id) in tests {
-            let location = context.function_meta(&func_id).name.location;
+        for (func_name, test_function) in tests {
+            let location = context.function_meta(&test_function.get_id()).name.location;
             let file_id = location.file;
 
             // Ignore diagnostics for any file that wasn't the file we saved

crates/nargo_cli/src/cli/test_cmd.rs

@@ -7,8 +7,7 @@ use nargo_toml::{get_package_manifest, resolve_workspace_from_toml, PackageSelec
 use noirc_driver::{compile_no_check, CompileOptions};
 use noirc_frontend::{
     graph::CrateName,
-    hir::{Context, FunctionNameMatch},
-    node_interner::FuncId,
+    hir::{def_map::TestFunction, Context, FunctionNameMatch},
 };
 use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
 
@@ -123,30 +122,56 @@ fn run_tests<B: Backend>(
 fn run_test<B: Backend>(
     backend: &B,
     test_name: &str,
-    main: FuncId,
+    test_function: TestFunction,
     context: &Context,
     show_output: bool,
     config: &CompileOptions,
 ) -> Result<(), CliError<B>> {
-    let mut program = compile_no_check(context, config, main).map_err(|err| {
+    let report_error = |err| {
         noirc_errors::reporter::report_all(&context.file_manager, &[err], config.deny_warnings);
-        CliError::Generic(format!("Test '{test_name}' failed to compile"))
-    })?;
-
-    // Note: We could perform this test using the unoptimized ACIR as generated by `compile_no_check`.
-    program.circuit = optimize_circuit(backend, program.circuit).unwrap().0;
-
-    // Run the backend to ensure the PWG evaluates functions like std::hash::pedersen,
-    // otherwise constraints involving these expressions will not error.
-    match execute_circuit(backend, program.circuit, WitnessMap::new(), show_output) {
-        Ok(_) => Ok(()),
-        Err(error) => {
-            let writer = StandardStream::stderr(ColorChoice::Always);
-            let mut writer = writer.lock();
-            writer.set_color(ColorSpec::new().set_fg(Some(Color::Red))).ok();
-            writeln!(writer, "failed").ok();
-            writer.reset().ok();
-            Err(error.into())
+        Err(CliError::Generic(format!("Test '{test_name}' failed to compile")))
+    };
+
+    let program = compile_no_check(context, config, test_function.get_id());
+    match program {
+        Ok(mut program) => {
+            // Note: We don't need to use the optimized ACIR here
+            program.circuit = optimize_circuit(backend, program.circuit).unwrap().0;
+
+            // Run the backend to ensure the PWG evaluates functions like std::hash::pedersen,
+            // otherwise constraints involving these expressions will not error.
+            let circuit_execution =
+                execute_circuit(backend, program.circuit, WitnessMap::new(), show_output);
+
+            if test_function.get_expect_failure_flag() {
+                match circuit_execution {
+                    Ok(_) => Err(CliError::Generic(format!("Test '{test_name}' should fail"))),
+                    Err(_) => Ok(()),
+                }
+            } else {
+                match circuit_execution {
+                    Ok(_) => Ok(()),
+                    Err(error) => {
+                        let writer = StandardStream::stderr(ColorChoice::Always);
+                        let mut writer = writer.lock();
+                        writer.set_color(ColorSpec::new().set_fg(Some(Color::Red))).ok();
+                        writeln!(writer, "failed").ok();
+                        writer.reset().ok();
+                        Err(error.into())
+                    }
+                }
+            }
+        }
+        Err(err) => {
+            if test_function.get_expect_failure_flag() {
+                if !err.diagnostic.message.contains("Failed constraint") {
+                    report_error(err)
+                } else {
+                    Ok(())
+                }
+            } else {
+                report_error(err)
+            }
         }
     }
 }

crates/noirc_frontend/src/ast/function.rs

@@ -83,7 +83,7 @@ impl From<FunctionDefinition> for NoirFunction {
         let kind = match fd.attribute {
             Some(Attribute::Builtin(_)) => FunctionKind::Builtin,
             Some(Attribute::Foreign(_)) => FunctionKind::LowLevel,
-            Some(Attribute::Test) => FunctionKind::Normal,
+            Some(Attribute::Test { .. }) => FunctionKind::Normal,
             Some(Attribute::Oracle(_)) => FunctionKind::Oracle,
             Some(Attribute::Deprecated(_)) | None => FunctionKind::Normal,
             Some(Attribute::Custom(_)) => FunctionKind::Normal,

crates/noirc_frontend/src/hir/def_map/mod.rs

@@ -3,7 +3,7 @@ use crate::hir::def_collector::dc_crate::DefCollector;
 use crate::hir::Context;
 use crate::node_interner::{FuncId, NodeInterner};
 use crate::parser::{parse_program, ParsedModule};
-use crate::token::Attribute;
+use crate::token::{Attribute, TestScope};
 use arena::{Arena, Index};
 use fm::{FileId, FileManager};
 use noirc_errors::{FileDiagnostic, Location};
@@ -129,12 +129,18 @@ impl CrateDefMap {
     pub fn get_all_test_functions<'a>(
         &'a self,
         interner: &'a NodeInterner,
-    ) -> impl Iterator<Item = FuncId> + 'a {
+    ) -> impl Iterator<Item = TestFunction> + 'a {
         self.modules.iter().flat_map(|(_, module)| {
-            module
-                .value_definitions()
-                .filter_map(|id| id.as_function())
-                .filter(|id| interner.function_meta(id).attributes == Some(Attribute::Test))
+            module.value_definitions().filter_map(|id| {
+                if let Some(func_id) = id.as_function() {
+                    match interner.function_meta(&func_id).attributes {
+                        Some(Attribute::Test(scope)) => Some(TestFunction::new(func_id, scope)),
+                        _ => None,
+                    }
+                } else {
+                    None
+                }
+            })
         })
     }
 
@@ -221,3 +227,25 @@ impl std::ops::IndexMut<LocalModuleId> for CrateDefMap {
         &mut self.modules[local_module_id.0]
     }
 }
+
+pub struct TestFunction {
+    id: FuncId,
+    scope: TestScope,
+}
+
+impl TestFunction {
+    fn new(id: FuncId, scope: TestScope) -> Self {
+        TestFunction { id, scope }
+    }
+
+    pub fn get_id(&self) -> FuncId {
+        self.id
+    }
+
+    pub fn get_expect_failure_flag(&self) -> bool {
+        match self.scope {
+            TestScope::ShouldFail => true,
+            TestScope::None => false,
+        }
+    }
+}

crates/noirc_frontend/src/hir/mod.rs

@@ -11,6 +11,8 @@ use def_map::{Contract, CrateDefMap};
 use fm::FileManager;
 use std::collections::HashMap;
 
+use self::def_map::TestFunction;
+
 /// Helper object which groups together several useful context objects used
 /// during name resolution. Once name resolution is finished, only the
 /// def_interner is required for type inference and monomorphization.
@@ -107,22 +109,22 @@ impl Context {
         &self,
         crate_id: &CrateId,
         pattern: FunctionNameMatch,
-    ) -> Vec<(String, FuncId)> {
+    ) -> Vec<(String, TestFunction)> {
         let interner = &self.def_interner;
         let def_map = self.def_map(crate_id).expect("The local crate should be analyzed already");
 
         def_map
             .get_all_test_functions(interner)
-            .filter_map(|id| {
-                let fully_qualified_name = self.fully_qualified_function_name(crate_id, &id);
+            .filter_map(|test_function| {
+                let fully_qualified_name =
+                    self.fully_qualified_function_name(crate_id, &test_function.get_id());
                 match &pattern {
-                    FunctionNameMatch::Anything => Some((fully_qualified_name, id)),
-                    FunctionNameMatch::Exact(pattern) => {
-                        (&fully_qualified_name == pattern).then_some((fully_qualified_name, id))
-                    }
-                    FunctionNameMatch::Contains(pattern) => {
-                        fully_qualified_name.contains(pattern).then_some((fully_qualified_name, id))
-                    }
+                    FunctionNameMatch::Anything => Some((fully_qualified_name, test_function)),
+                    FunctionNameMatch::Exact(pattern) => (&fully_qualified_name == pattern)
+                        .then_some((fully_qualified_name, test_function)),
+                    FunctionNameMatch::Contains(pattern) => fully_qualified_name
+                        .contains(pattern)
+                        .then_some((fully_qualified_name, test_function)),
                 }
             })
             .collect()

crates/noirc_frontend/src/hir/resolution/resolver.rs

@@ -700,7 +700,8 @@
             self.push_err(ResolverError::DistinctNotAllowed { ident: func.name_ident().clone() });
         }
 
-        if attributes == Some(Attribute::Test) && !parameters.is_empty() {
+        if matches!(attributes, Some(Attribute::Test { .. })) && !parameters.is_empty()
+        {
             self.push_err(ResolverError::TestFunctionHasParameters {
                 span: func.name_ident().span(),
             });
@@ -1934,7 +1935,7 @@
                 println(f"I want to print {0}");
 
                 let new_val = 10;
                 println(f"randomstring{new_val}{new_val}");
             }
             fn println<T>(x : T) -> T {
                 x

crates/noirc_frontend/src/lexer/lexer.rs

@@ -1,3 +1,6 @@
+#[cfg(test)]
+use crate::token::TestScope;
+
 use super::{
     errors::LexerErrorKind,
     token::{Attribute, IntType, Keyword, SpannedToken, Token, Tokens},
@@ -462,6 +465,43 @@ fn custom_attribute() {
     assert_eq!(token.token(), &Token::Attribute(Attribute::Custom("custom(hello)".to_string())));
 }
 
+#[test]
+fn test_attribute() {
+    let input = r#"#[test]"#;
+    let mut lexer = Lexer::new(input);
+
+    let token = lexer.next().unwrap().unwrap();
+    assert_eq!(token.token(), &Token::Attribute(Attribute::Test(TestScope::None)));
+}
+#[test]
+fn test_attribute_with_valid_scope() {
+    let input = r#"#[test(should_fail)]"#;
+    let mut lexer = Lexer::new(input);
+
+    let token = lexer.next().unwrap().unwrap();
+    assert_eq!(token.token(), &Token::Attribute(Attribute::Test(TestScope::ShouldFail)));
+}
+
+#[test]
+fn test_attribute_with_invalid_scope() {
+    let input = r#"#[test(invalid_scope)]"#;
+    let mut lexer = Lexer::new(input);
+
+    let token = lexer.next().unwrap();
+    let err = match token {
+        Ok(_) => panic!("test has an invalid scope, so expected an error"),
+        Err(err) => err,
+    };
+
+    // Check if error is MalformedFuncAttribute and found is "foo"
+    let sub_string = match err {
+        LexerErrorKind::MalformedFuncAttribute { found, .. } => found,
+        _ => panic!("expected malformed func attribute error"),
+    };
+
+    assert_eq!(sub_string, "test(invalid_scope)");
+}
+
 #[test]
 fn test_custom_gate_syntax() {
     let input = "#[foreign(sha256)]#[foreign(blake2s)]#[builtin(sum)]";

crates/noirc_frontend/src/lexer/token.rs

@@ -316,6 +316,36 @@
     }
 }
 
+/// TestScope is used to specify additional annotations for test functions
+#[derive(PartialEq, Eq, Hash, Debug, Clone, PartialOrd, Ord)]
+pub enum TestScope {
+    /// If a test has a scope of ShouldFail, then it is expected to fail
+    ShouldFail,
+    /// No scope is applied and so the test must pass
+    None,
+}
+
+impl TestScope {
+    fn lookup_str(string: &str) -> Option<TestScope> {
+        match string {
+            "should_fail" => Some(TestScope::ShouldFail),
+            _ => None,
+        }
+    }
+    fn as_str(&self) -> &'static str {
+        match self {
+            TestScope::ShouldFail => "should_fail",
+            TestScope::None => "",
+        }
+    }
+}
+
+impl fmt::Display for TestScope {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "({})", self.as_str())
+    }
+}
+
 #[derive(PartialEq, Eq, Hash, Debug, Clone, PartialOrd, Ord)]
 // Attributes are special language markers in the target language
 // An example of one is `#[SHA256]` . Currently only Foreign attributes are supported
@@ -325,17 +355,17 @@
     Builtin(String),
     Oracle(String),
     Deprecated(Option<String>),
-    Test,
+    Test(TestScope),
     Custom(String),
 }
 
 impl fmt::Display for Attribute {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match *self {
+        match self {
             Attribute::Foreign(ref k) => write!(f, "#[foreign({k})]"),
             Attribute::Builtin(ref k) => write!(f, "#[builtin({k})]"),
             Attribute::Oracle(ref k) => write!(f, "#[oracle({k})]"),
-            Attribute::Test => write!(f, "#[test]"),
+            Attribute::Test(scope) => write!(f, "#[test{}]", scope),
             Attribute::Deprecated(None) => write!(f, "#[deprecated]"),
             Attribute::Deprecated(Some(ref note)) => write!(f, r#"#[deprecated("{note}")]"#),
             Attribute::Custom(ref k) => write!(f, "#[{k}]"),
@@ -391,7 +421,16 @@
 
                 Attribute::Deprecated(name.trim_matches('"').to_string().into())
             }
-            ["test"] => Attribute::Test,
+            ["test"] => Attribute::Test(TestScope::None),
+            ["test", name] => {
+                validate(name)?;
+                let malformed_scope =
+                    LexerErrorKind::MalformedFuncAttribute { span, found: word.to_owned() };
+                match TestScope::lookup_str(name) {
+                    Some(scope) => Attribute::Test(scope),
+                    None => return Err(malformed_scope),
+                }
+            }
             tokens => {
                 tokens.iter().try_for_each(|token| validate(token))?;
                 Attribute::Custom(word.to_owned())
@@ -431,7 +470,7 @@
             Attribute::Builtin(string) => string,
             Attribute::Oracle(string) => string,
             Attribute::Deprecated(Some(string)) => string,
-            Attribute::Test | Attribute::Deprecated(None) => "",
+            Attribute::Test { .. } | Attribute::Deprecated(None) => "",
             Attribute::Custom(string) => string,
         }
     }
@@ -496,7 +535,7 @@
             Keyword::Field => write!(f, "Field"),
             Keyword::Fn => write!(f, "fn"),
             Keyword::For => write!(f, "for"),
             Keyword::FormatString => write!(f, "fmtstr"),
             Keyword::Global => write!(f, "global"),
             Keyword::If => write!(f, "if"),
             Keyword::Impl => write!(f, "impl"),
@@ -538,7 +577,7 @@
             "Field" => Keyword::Field,
             "fn" => Keyword::Fn,
             "for" => Keyword::For,
             "fmtstr" => Keyword::FormatString,
             "global" => Keyword::Global,
             "if" => Keyword::If,
             "impl" => Keyword::Impl,