diff --git a/crates/mdbook-quiz-validate/src/impls/markdown.rs b/crates/mdbook-quiz-validate/src/impls/markdown.rs
index abe97a3..673bfb6 100644
--- a/crates/mdbook-quiz-validate/src/impls/markdown.rs
+++ b/crates/mdbook-quiz-validate/src/impls/markdown.rs
@@ -76,7 +76,7 @@ prompt.prompt = "Hello **world**"
 answer.answer = ""
 prompt.distractors = [""]
 "#;
-  assert!(crate::harness(contents).is_ok());
+  assert!(crate::test::harness(contents).is_ok());
 }
 
 #[test]
@@ -89,5 +89,5 @@ answer.answer = ""
 prompt.distractors = [""]
 "#;
   // TODO: right now this test is just verified looking at stderr
-  assert!(crate::harness(contents).is_ok());
+  assert!(crate::test::harness(contents).is_ok());
 }
diff --git a/crates/mdbook-quiz-validate/src/impls/mod.rs b/crates/mdbook-quiz-validate/src/impls/mod.rs
index ff80c72..5e5ddb9 100644
--- a/crates/mdbook-quiz-validate/src/impls/mod.rs
+++ b/crates/mdbook-quiz-validate/src/impls/mod.rs
@@ -95,7 +95,7 @@ prompt.prompt = ""
 answer.answer = ""
 prompt.distractors = [""]
 "#;
-  assert!(crate::harness(contents).is_err());
+  assert!(crate::test::harness(contents).is_err());
 }
 
 #[test]
@@ -116,7 +116,7 @@ prompt.distractors = [""]
 "#;
 
   // TODO: right now this test is just verified looking at stderr
-  assert!(crate::harness(contents).is_ok());
+  assert!(crate::test::harness(contents).is_ok());
 }
 
 #[test]
@@ -134,5 +134,5 @@ prompt.prompt = ""
 answer.answer = ""
 prompt.distractors = [""]
 "#;
-  assert!(crate::harness(contents).is_err());
+  assert!(crate::test::harness(contents).is_err());
 }
diff --git a/crates/mdbook-quiz-validate/src/impls/multiple_choice.rs b/crates/mdbook-quiz-validate/src/impls/multiple_choice.rs
index 40dedc6..9375a65 100644
--- a/crates/mdbook-quiz-validate/src/impls/multiple_choice.rs
+++ b/crates/mdbook-quiz-validate/src/impls/multiple_choice.rs
@@ -73,7 +73,7 @@ prompt.prompt = ""
 answer.answer = ""
 prompt.distractors = [""]
 "#;
-  assert!(crate::harness(contents).is_ok());
+  assert!(crate::test::harness(contents).is_ok());
 }
 
 #[test]
@@ -87,5 +87,5 @@ prompt.distractors = [""]
 prompt.answerIndex = 0
 prompt.sortAnswers = true
 "#;
-  assert!(crate::harness(contents).is_err());
+  assert!(crate::test::harness(contents).is_err());
 }
diff --git a/crates/mdbook-quiz-validate/src/impls/tracing.rs b/crates/mdbook-quiz-validate/src/impls/tracing.rs
index a1da171..f569f21 100644
--- a/crates/mdbook-quiz-validate/src/impls/tracing.rs
+++ b/crates/mdbook-quiz-validate/src/impls/tracing.rs
@@ -107,7 +107,7 @@ fn main() {
 answer.doesCompile = true
 answer.stdout = "Hello world"
 "#;
-  assert!(crate::harness(contents).is_ok());
+  assert!(crate::test::harness(contents).is_ok());
 }
 
 #[test]
@@ -123,7 +123,7 @@ fn main() {
 answer.doesCompile = true
 answer.stdout = ""
 "#;
-  assert!(crate::harness(contents).is_err());
+  assert!(crate::test::harness(contents).is_err());
 }
 
 #[test]
@@ -139,5 +139,5 @@ fn main() {
 answer.doesCompile = true
 answer.stdout = "meep meep"
 "#;
-  assert!(crate::harness(contents).is_err());
+  assert!(crate::test::harness(contents).is_err());
 }
diff --git a/crates/mdbook-quiz-validate/src/lib.rs b/crates/mdbook-quiz-validate/src/lib.rs
index 9dfef88..7a66c1b 100644
--- a/crates/mdbook-quiz-validate/src/lib.rs
+++ b/crates/mdbook-quiz-validate/src/lib.rs
@@ -22,8 +22,15 @@ pub use toml_spanned_value::SpannedValue;
 mod impls;
 mod spellcheck;
 
-/// A thread-safe mutable set of question identifiers.
-pub type IdSet = Arc<Mutex<HashSet<String>>>;
+#[derive(Default)]
+struct ValidatedInner {
+  ids: HashSet<String>,
+  paths: HashSet<PathBuf>,
+}
+
+#[derive(Default, Clone)]
+/// A thread-safe mutable set of already-validated identifiers and paths.
+pub struct Validated(Arc<Mutex<ValidatedInner>>);
 
 struct QuizDiagnostic {
   error: miette::Error,
@@ -34,17 +41,17 @@ pub(crate) struct ValidationContext {
   diagnostics: RefCell<Vec<QuizDiagnostic>>,
   path: PathBuf,
   contents: String,
-  ids: IdSet,
+  validated: Validated,
   spellcheck: bool,
 }
 
 impl ValidationContext {
-  pub fn new(path: &Path, contents: &str, ids: IdSet, spellcheck: bool) -> Self {
+  pub fn new(path: &Path, contents: &str, validated: Validated, spellcheck: bool) -> Self {
     ValidationContext {
       diagnostics: Default::default(),
       path: path.to_owned(),
       contents: contents.to_owned(),
-      ids,
+      validated,
       spellcheck,
     }
   }
@@ -71,7 +78,7 @@ impl ValidationContext {
   }
 
   pub fn check_id(&mut self, id: &str, value: &SpannedValue) {
-    let new_id = self.ids.lock().unwrap().insert(id.to_string());
+    let new_id = self.validated.0.lock().unwrap().ids.insert(id.to_string());
     if !new_id {
       self.error(miette!(
         labels = vec![value.labeled_span()],
@@ -149,8 +156,18 @@ struct ParseError {
 }
 
 /// Runs validation on a quiz with TOML-format `contents` at `path` under the ID set `ids`.
-pub fn validate(path: &Path, contents: &str, ids: &IdSet, spellcheck: bool) -> anyhow::Result<()> {
-  let mut cx = ValidationContext::new(path, contents, Arc::clone(ids), spellcheck);
+pub fn validate(
+  path: &Path,
+  contents: &str,
+  validated: &Validated,
+  spellcheck: bool,
+) -> anyhow::Result<()> {
+  let not_checked = validated.0.lock().unwrap().paths.insert(path.to_path_buf());
+  if !not_checked {
+    return Ok(());
+  }
+
+  let mut cx = ValidationContext::new(path, contents, validated.clone(), spellcheck);
 
   let parse_result = toml::from_str::<Quiz>(contents);
   match parse_result {
@@ -180,14 +197,29 @@ pub fn validate(path: &Path, contents: &str, ids: &IdSet, spellcheck: bool) -> a
 }
 
 #[cfg(test)]
-pub(crate) fn harness(contents: &str) -> anyhow::Result<()> {
-  validate(Path::new("dummy.rs"), contents, &IdSet::default(), true)
-}
-
-#[cfg(test)]
-mod test {
+pub(crate) mod test {
   use super::*;
 
+  pub(crate) fn harness(contents: &str) -> anyhow::Result<()> {
+    validate(Path::new("dummy.rs"), contents, &Validated::default(), true)
+  }
+
+  #[test]
+  fn validate_twice() -> anyhow::Result<()> {
+    let contents = r#"
+[[questions]]
+id = "foobar"
+type = "MultipleChoice"
+prompt.prompt = ""
+answer.answer = ""
+prompt.distractors = [""]
+"#;
+    let validated = Validated::default();
+    validate(Path::new("dummy.rs"), contents, &validated, true)?;
+    validate(Path::new("dummy.rs"), contents, &validated, true)?;
+    Ok(())
+  }
+
   #[test]
   fn validate_parse_error() {
     let contents = r#"
diff --git a/crates/mdbook-quiz/src/main.rs b/crates/mdbook-quiz/src/main.rs
index c060b8f..c28aa82 100644
--- a/crates/mdbook-quiz/src/main.rs
+++ b/crates/mdbook-quiz/src/main.rs
@@ -3,7 +3,7 @@ use mdbook_preprocessor_utils::{
   mdbook::preprocess::PreprocessorContext, Asset, SimplePreprocessor,
 };
 
-use mdbook_quiz_validate::IdSet;
+use mdbook_quiz_validate::Validated;
 use regex::Regex;
 use std::{
   env,
@@ -61,7 +61,7 @@ struct QuizConfig {
 
 struct QuizPreprocessor {
   config: QuizConfig,
-  question_ids: IdSet,
+  validated: Validated,
   #[cfg(feature = "aquascope")]
   aquascope: mdbook_aquascope::AquascopePreprocessor,
 }
@@ -132,7 +132,7 @@ impl QuizPreprocessor {
     mdbook_quiz_validate::validate(
       &quiz_path_abs,
       &content_toml,
-      &self.question_ids,
+      &self.validated,
       self.config.spellcheck.unwrap_or(false),
     )?;
 
@@ -215,7 +215,7 @@ impl SimplePreprocessor for QuizPreprocessor {
 
     Ok(QuizPreprocessor {
       config,
-      question_ids: IdSet::default(),
+      validated: Validated::default(),
       #[cfg(feature = "aquascope")]
       aquascope: mdbook_aquascope::AquascopePreprocessor::new()
         .context("Aquascope failed to initialize")?,