Let lt-proc -b handle special ANY_CHAR tag (<w/> from lsx)

apertium · Dec 19, 2024 · 51898cc · 51898cc
1 parent 716a00f
commit 51898cc
Show file tree

Hide file tree

Showing 5 changed files with 59 additions and 4 deletions.
diff --git a/lttoolbox/fst_processor.cc b/lttoolbox/fst_processor.cc
@@ -627,6 +627,8 @@ void
 FSTProcessor::load(FILE *input)
 {
   readTransducerSet(input, alphabetic_chars, alphabet, transducers);
+  alphabet.includeSymbol("<ANY_CHAR>"_u);
+  any_char = alphabet("<ANY_CHAR>"_u);
 }
 
 void
@@ -1755,7 +1757,14 @@ FSTProcessor::bilingual(InputFile& input, UFILE *output, GenerationMode mode)
     if (reader.readings[index].mark == '#') current_state.step('#');
     for (size_t i = 0; i < symbols.size(); i++) {
       seenTags = seenTags || alphabet.isTag(symbols[i]);
-      current_state.step_case(symbols[i], beCaseSensitive(current_state));
+      UString source;
+      alphabet.getSymbol(source, symbols[i]);
+      if(beCaseSensitive(current_state)) { // allow any_char
+        current_state.step_override(symbols[i], any_char, symbols[i]);
+      }
+      else {                    // include lower alt
+        current_state.step_override(symbols[i], towlower(symbols[i]), any_char, symbols[i]);
+      }
       if (current_state.isFinal(all_finals)) {
         queue_start = i;
         current_state.filterFinalsArray(result,

diff --git a/lttoolbox/fst_processor.h b/lttoolbox/fst_processor.h
@@ -252,6 +252,11 @@ class FSTProcessor
    */
   int maxWeightClasses = INT_MAX;
 
+  /**
+   * The alphabet index of the tag <ANY_CHAR>
+   */
+  int any_char;
+
   /**
    * Prints an error of input stream and exits
    */

diff --git a/lttoolbox/state.h b/lttoolbox/state.h
@@ -113,9 +113,9 @@ class State
 
   /**
    * Make a transition, but overriding the output symbol
-   * @param input symbol
-   * @param output symbol we expect to appear
-   * @param output symbol we want to appear
+   * @param input symbol read from infile
+   * @param output symbol from the FST
+   * @param output symbol we want to appear in outfile
    */
   void apply_override(int const input, int const old_sym, int const new_sym);
 

diff --git a/tests/data/pass-through.lsx b/tests/data/pass-through.lsx
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<dictionary type="separable">
+  <alphabet></alphabet>
+  <sdefs>
+    <sdef n="MERGED"/>
+  </sdefs>
+
+  <pardefs>
+    <pardef n="foo">
+      <e>   <i>foo<d/></i>            </e>
+    </pardef>
+  </pardefs>
+
+  <section id="main" type="standard">
+
+    <e c="pass-through MERGED words">
+      <i><w/><s n="MERGED"/></i>
+    </e>
+  </section>
+</dictionary>
diff --git a/tests/lt_proc/__init__.py b/tests/lt_proc/__init__.py
@@ -479,6 +479,11 @@ class BiltransGarbage(ProcTest):
     inputs = ['^$']
     expectedOutputs = ['^$']
 
+class BiltransSimple(ProcTest):
+    procflags = ['-b', '-z']
+    inputs = ['^abc$']
+    expectedOutputs = ['^abc/ab<n><def>$']
+
 class SlashesInTags(ProcTest):
     procdix = 'data/slash-tags.dix'
     procflags = ['-b', '-z']
@@ -496,5 +501,21 @@ class SlashesInTags(ProcTest):
                        '^\\*lobwana1.1<n><1/2><a/b>/*lopwana1.1<n><1/2><a/b>$',
                        '^\\*lobwana1.1<n><3/4><a/b>/@\\*lobwana1.1<n><3/4><a/b>$']
 
+class BiltransAnyChar(ProcTest):
+    procdix = 'data/pass-through.lsx'
+    procflags = ['-b', '-z']
+    # Using r'' to avoid doubling escapes even more:
+    inputs = [r'^simple<MERGED>$']
+    expectedOutputs = [r'^simple<MERGED>/simple<MERGED>$']
+
+
+class BiltransAnyCharEscapes(ProcTest):
+    procdix = 'data/pass-through.lsx'
+    procflags = ['-b', '-z']
+    # Using r'' to avoid doubling escapes even more:
+    inputs = [r'^«\[\[tf:i:a\]\]s\\\^å\[\[\/\]\]»<MERGED>$']
+    expectedOutputs = [r'^«\[\[tf:i:a\]\]s\\\^å\[\[\/\]\]»<MERGED>/«\[\[tf:i:a\]\]s\\\^å\[\[\/\]\]»<MERGED>$']
+
+
 # These fail on some systems:
 #from null_flush_invalid_stream_format import *