feat: add distinct references (#127)

* feat: add citation element

* feat: add preamble parsing

* feat: allow multiple csl locale files

* fix: only merge preamble if a preamble exists

* add citeproc-js + rendering of the in-text part of citations

* prepare citeproc adapter for footnotes and bibliography

* use values from preamble + add some styles and locales

* add bibliography and footnotes to end of document

* replace String::from with .to_string()

* support german and english for bibliography heading

* add the csl struct that Manuel generated

* merge input .csl files to one json object

* change type of citation_locales to HashMap<Locale, PathBuf>

* add warnings to reading the locale and style files

* add CiteErrors

* run cargo fmt

* move json serialization from get_citation_strings to new of Context + add logs when it fails

* serialize each citation ids vector individually

* add class for Css entries

* try to stop clippy from complaining about the generated file

* parse csl-locales from cli using a vector of tuples

* serialize locale-pathbuf map using map serializer from serde

* use custom visitor for citation_locales deserialization

* use custom visitor for multiple locales deserialization

* run cargo fmt

* Revert "Merge branch 'main' into cite-box"

This reverts commit 4c5eeb0, reversing
changes made to 9d064a9.

* Revert "Revert "Merge branch 'main' into cite-box""

This reverts commit 2746775.

* fix: resolve merge conflicts

* fix: add bib & footnotes to umi

* fix: pass output format to render context

* add unit test for citations

* don't print bibliography and footnotes if the strings are empty

* don't start the citeproc processor if the input doesn't contain citations

* parse distinct references

* parse distinct references using own parser method

* also parse dots in parser for distinct references

* add (simplified) rendering of distinct references

* print integers without .0

* use author-only of citeproc js for authors in distinct references

* fix formatting + tests

* add unit tests for rendering distinct references

* fix: add dot as field separator

* fix: move dist-ref parser to scoped-parser

---------

Co-authored-by: Manuel Hatzl <[email protected]>

inline/src/element/mod.rs

@@ -1,5 +1,6 @@
 //! Contains all Unimarkup [`Inline`] elements.
 
+use crate::element::substitution::DistinctReference;
 use unimarkup_commons::{
     lexer::{position::Position, span::Span},
     parsing::Element,
@@ -65,7 +66,7 @@ pub enum Inline {
     /// Underlined content.
     Underline(Underline),
 
-    /// Content in a subscript.   
+    /// Content in a subscript.
     Subscript(Subscript),
 
     /// Content in a superscript.
@@ -125,6 +126,9 @@ pub enum Inline {
 
     /// Direct URI
     DirectUri(DirectUri),
+
+    /// Distinct reference
+    DistinctReference(DistinctReference),
 }
 
 impl Inline {
@@ -157,6 +161,7 @@ impl Inline {
             Inline::DirectUri(_) => "DirectUri",
             Inline::NamedSubstitution(_) => "NamedSubstitution",
             Inline::ImplicitSubstitution(_) => "ImplicitSubstitution",
+            Inline::DistinctReference(_) => "DistinctReference",
         }
     }
 }
@@ -186,6 +191,7 @@ impl InlineElement for Inline {
             Inline::EscapedPlain(inline) => inline.as_unimarkup(),
             Inline::DirectUri(inline) => inline.as_unimarkup(),
             Inline::ImplicitSubstitution(inline) => inline.as_unimarkup(),
+            Inline::DistinctReference(inline) => inline.as_unimarkup(),
 
             Inline::NamedSubstitution(_) => todo!(),
         }
@@ -215,6 +221,7 @@ impl InlineElement for Inline {
             Inline::EscapedPlain(inline) => inline.start(),
             Inline::DirectUri(inline) => inline.start(),
             Inline::ImplicitSubstitution(inline) => inline.start(),
+            Inline::DistinctReference(inline) => inline.start(),
 
             Inline::NamedSubstitution(_) => todo!(),
         }
@@ -244,6 +251,7 @@ impl InlineElement for Inline {
             Inline::EscapedPlain(inline) => inline.end(),
             Inline::DirectUri(inline) => inline.end(),
             Inline::ImplicitSubstitution(inline) => inline.end(),
+            Inline::DistinctReference(inline) => inline.end(),
 
             Inline::NamedSubstitution(_) => todo!(),
         }

inline/src/element/substitution/mod.rs

@@ -1,6 +1,9 @@
+use crate::parser::InlineParser;
+use crate::InlineTokenKind;
+use unimarkup_commons::lexer::token::iterator::PeekingNext;
 use unimarkup_commons::lexer::{position::Position, token::implicit::ImplicitSubstitutionKind};
 
-use super::InlineElement;
+use super::{Inline, InlineElement};
 
 pub mod named;
 
@@ -69,3 +72,107 @@ impl InlineElement for DirectUri {
         self.end
     }
 }
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DistinctReference {
+    id: String,
+    fields: Vec<String>,
+    start: Position,
+    end: Position,
+}
+
+impl DistinctReference {
+    pub fn new(id: String, fields: Vec<String>, start: Position, end: Position) -> Self {
+        Self {
+            id,
+            fields,
+            start,
+            end,
+        }
+    }
+
+    pub fn id(&self) -> &str {
+        &self.id
+    }
+
+    pub fn fields(&self) -> &Vec<String> {
+        &self.fields
+    }
+}
+
+impl InlineElement for DistinctReference {
+    fn as_unimarkup(&self) -> String {
+        format!("{}.{}", self.id.clone(), self.fields.clone().join("."))
+    }
+
+    fn start(&self) -> unimarkup_commons::lexer::position::Position {
+        self.start
+    }
+
+    fn end(&self) -> unimarkup_commons::lexer::position::Position {
+        self.end
+    }
+}
+
+// Parses distinct reference
+pub(crate) fn parse_distinct_reference<'s, 'i>(
+    mut parser: InlineParser<'s, 'i>,
+) -> (InlineParser<'s, 'i>, Option<Inline>) {
+    let open_token_opt = parser.iter.peeking_next(|_| true);
+    if open_token_opt.is_none() {
+        return (parser, None);
+    }
+
+    let open_token = open_token_opt.expect("Checked above to be not None.");
+
+    parser.iter.next(); // consume open token => now it will lead to Some(inline)
+
+    let mut parsed_token_strings = Vec::new();
+    let mut entries = Vec::new();
+
+    while let Some(kind) = parser.iter.peek_kind() {
+        let token = parser.iter.next().unwrap();
+        if kind == InlineTokenKind::Eoi || kind == InlineTokenKind::Cite {
+            break;
+        }
+        if kind == InlineTokenKind::Dot || token.as_str() == "." {
+            entries.push(parsed_token_strings.join(""));
+            parsed_token_strings = Vec::new();
+        } else {
+            parsed_token_strings.push(token.as_str().to_string());
+        }
+    }
+
+    let parsed_str = parsed_token_strings.join("");
+    if !parsed_str.is_empty() {
+        entries.push(parsed_str);
+    }
+
+    if entries.len() < 2 {
+        entries.push("authors".to_string());
+    }
+    let id = entries[0].clone();
+    entries.remove(0);
+
+    let prev_token = parser
+        .iter
+        .prev_token()
+        .expect("Previous token must exist, because peek above would else have returned None.");
+
+    let end = if parser.iter.end_reached() {
+        //TODO: Check for optional attributes here
+        prev_token.end
+    } else {
+        crate::element::helper::implicit_end_using_prev(&prev_token)
+    };
+
+    (
+        parser,
+        Some(Inline::DistinctReference(DistinctReference::new(
+            id,
+            entries,
+            open_token.start,
+            end,
+        ))),
+    )
+}

inline/src/parser.rs

@@ -220,6 +220,9 @@ fn get_scoped_parser(kind: InlineTokenKind, logic_only: bool) -> Option<InlinePa
             Some(crate::element::formatting::scoped::parse_math)
         }
         InlineTokenKind::OpenBracket if !logic_only => Some(crate::element::textbox::parse),
+        InlineTokenKind::Cite if !logic_only => {
+            Some(crate::element::substitution::parse_distinct_reference)
+        }
         _ => None,
     }
 }

inline/src/tokenize/kind.rs

@@ -243,7 +243,10 @@ impl InlineTokenKind {
     pub fn is_scoped_format_keyword(&self) -> bool {
         matches!(
             self,
-            InlineTokenKind::Verbatim | InlineTokenKind::Math | InlineTokenKind::NamedSubstitution
+            InlineTokenKind::Verbatim
+                | InlineTokenKind::Math
+                | InlineTokenKind::NamedSubstitution
+                | InlineTokenKind::Cite
         )
     }
 }

inline/tests/parser/snapshot.rs

@@ -128,6 +128,7 @@ fn inner_snapshot(inline: &Inline) -> String {
 
         Inline::NamedSubstitution(_) => todo!(),
         Inline::ImplicitSubstitution(impl_subst) => impl_subst.subst().to_string(),
+        Inline::DistinctReference(inline) => inline.as_unimarkup(),
     }
 }
 

inline/tests/spec/markup/distinct_reference.yml

@@ -0,0 +1,34 @@
+# Unimarkup specification version
+spec: "0.0.1"
+
+name: distinct_reference
+description: Test lexing and parsing of text with distinct reference inline formatting.
+
+tests:
+  - name: distinct-reference-without-fields
+    description: |
+      Distinct reference where the field "authors" should be added.
+
+    input: |
+      Text with distinct reference &&id-1&& without fields.
+
+  - name: distinct-reference-with-one-field
+    description: |
+      Distinct reference with one field.
+
+    input: |
+      Text with distinct reference &&id-1.title&& with field title.
+
+  - name: distinct-reference-with-multiple-fields
+    description: |
+      Distinct reference with multiple fields.
+
+    input: |
+      Text with distinct reference &&id-1.author.0.family&& with fields author, 0 and family.
+
+  - name: multiple-distinct-references
+    description: |
+      Two distinct references
+
+    input: |
+      Text with a distinct reference &&id-1.title&& and another one &&id-2.issued&&.

inline/tests/spec/snapshots/parser/distinct_reference/distinct-reference-with-multiple-fields.snap

@@ -0,0 +1,22 @@
+---
+source: inline/tests/parser/mod.rs
+info: "Test 'distinct-reference-with-multiple-fields' from 'markup\\distinct_reference.yml'"
+---
+Plain @ (1:1)->(1:30) (
+    Text with distinct reference 
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+)
+DistinctReference @ (1:30)->(1:54) (
+    id-1.author.0.family
+)
+Plain @ (1:54)->(1:88) (
+     with fields author, 0 and family.
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+)
+
+---
+With input:
+
+Text with distinct reference &&id-1.author.0.family&& with fields author, 0 and family.
+
+

inline/tests/spec/snapshots/parser/distinct_reference/distinct-reference-with-one-field.snap

@@ -0,0 +1,22 @@
+---
+source: inline/tests/parser/mod.rs
+info: "Test 'distinct-reference-with-one-field' from 'markup\\distinct_reference.yml'"
+---
+Plain @ (1:1)->(1:30) (
+    Text with distinct reference 
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+)
+DistinctReference @ (1:30)->(1:44) (
+    id-1.title
+)
+Plain @ (1:44)->(1:62) (
+     with field title.
+    ^^^^^^^^^^^^^^^^^^
+)
+
+---
+With input:
+
+Text with distinct reference &&id-1.title&& with field title.
+
+

inline/tests/spec/snapshots/parser/distinct_reference/distinct-reference-without-fields.snap

@@ -0,0 +1,22 @@
+---
+source: inline/tests/parser/mod.rs
+info: "Test 'distinct-reference-without-fields' from 'markup\\distinct_reference.yml'"
+---
+Plain @ (1:1)->(1:30) (
+    Text with distinct reference 
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+)
+DistinctReference @ (1:30)->(1:38) (
+    id-1.authors
+)
+Plain @ (1:38)->(1:54) (
+     without fields.
+    ^^^^^^^^^^^^^^^^
+)
+
+---
+With input:
+
+Text with distinct reference &&id-1&& without fields.
+
+

inline/tests/spec/snapshots/parser/distinct_reference/multiple-distinct-references.snap

@@ -0,0 +1,29 @@
+---
+source: inline/tests/parser/mod.rs
+info: "Test 'multiple-distinct-references' from 'markup\\distinct_reference.yml'"
+---
+Plain @ (1:1)->(1:32) (
+    Text with a distinct reference 
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+)
+DistinctReference @ (1:32)->(1:46) (
+    id-1.title
+)
+Plain @ (1:46)->(1:63) (
+     and another one 
+    ^^^^^^^^^^^^^^^^^
+)
+DistinctReference @ (1:63)->(1:78) (
+    id-2.issued
+)
+Plain @ (1:78)->(1:79) (
+    .
+    ^
+)
+
+---
+With input:
+
+Text with a distinct reference &&id-1.title&& and another one &&id-2.issued&&.
+
+

render/src/html/citeproc/js/citeproc_adapter.js

@@ -182,6 +182,20 @@ export function getCitationStrings(citationIds) {
     return citationResults;
 }
 
+export function getAuthorOnly(citeId) {
+    const citation = {
+        properties: {
+            noteIndex: 0
+        },
+        citationItems: [{
+            id: citeId,
+            "author-only": true
+        }],
+        citationID: citeId
+    };
+    return citeproc.processCitationCluster(citation, citationsPre, citationsPost)[1][0][1];
+}
+
 export function hasFootnotes() {
     return footnoteResults.length > 0;
 }