added jsgf support for tags

include/sphinxbase/fsg_model.h

@@ -8,24 +8,24 @@
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer. 
+ *    notice, this list of conditions and the following disclaimer.
  *
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in
  *    the documentation and/or other materials provided with the
  *    distribution.
  *
  *
- * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
- * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
  * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * ====================================================================
@@ -34,7 +34,7 @@
 
 /*
  * fsg_model.h -- Word-level finite state graph
- * 
+ *
  * **********************************************
  * CMU ARPA Speech Project
  *
@@ -68,6 +68,8 @@ extern "C" {
 }
 #endif
 
+#define MAX_TAG_SIZE 50
+
 /*
  * A single transition in the FSG.
  */
@@ -76,12 +78,14 @@ typedef struct fsg_link_s {
     int32 to_state;
     int32 logs2prob;	/**< log(transition probability)*lw */
     int32 wid;		/**< Word-ID; <0 if epsilon or null transition */
+    char tag[MAX_TAG_SIZE];   /**< List containig tags (char *) associated with the link */
 } fsg_link_t;
 
 /* Access macros */
 #define fsg_link_from_state(l)	((l)->from_state)
 #define fsg_link_to_state(l)	((l)->to_state)
 #define fsg_link_wid(l)		((l)->wid)
+#define fsg_link_tag(l)		((l)->tag)
 #define fsg_link_logs2prob(l)	((l)->logs2prob)
 
 /**
@@ -154,9 +158,9 @@ fsg_model_t *fsg_model_init(char const *name, logmath_t *lmath,
 /**
  * Read a word FSG from the given file and return a pointer to the structure
  * created.  Return NULL if any error occurred.
- * 
+ *
  * File format:
- * 
+ *
  * <pre>
  *   Any number of comment lines; ignored
  *   FSG_BEGIN [<fsgname>]
@@ -169,24 +173,24 @@ fsg_model_t *fsg_model_init(char const *name, logmath_t *lmath,
  *   FSG_END
  *   Any number of comment lines; ignored
  * </pre>
- * 
+ *
  * The FSG spec begins with the line containing the keyword FSG_BEGIN.
  * It has an optional fsg name string.  If not present, the FSG has the empty
  * string as its name.
- * 
+ *
  * Following the FSG_BEGIN declaration is the number of states, the start
  * state, and the final state, each on a separate line.  States are numbered
  * in the range [0 .. <numberofstate>-1].
- * 
+ *
  * These are followed by all the state transitions, each on a separate line,
  * and terminated by the FSG_END line.  A state transition has the given
  * probability of being taken, and emits the given word.  The word emission
  * is optional; if word-string omitted, it is an epsilon or null transition.
- * 
+ *
  * Comments can also be embedded within the FSG body proper (i.e. between
  * FSG_BEGIN and FSG_END): any line with a # character in col 1 is treated
  * as a comment line.
- * 
+ *
  * Return value: a new fsg_model_t structure if the file is successfully
  * read, NULL otherwise.
  */
@@ -239,7 +243,7 @@ int fsg_model_word_id(fsg_model_t *fsg, char const *word);
  */
 SPHINXBASE_EXPORT
 void fsg_model_trans_add(fsg_model_t * fsg,
-                         int32 from, int32 to, int32 logp, int32 wid);
+                         int32 from, int32 to, int32 logp, int32 wid, glist_t tags);
 
 /**
  * Add a null transition between the given states.

include/sphinxbase/jsgf.h

@@ -63,6 +63,10 @@ extern "C" {
 
 typedef struct jsgf_s jsgf_t;
 typedef struct jsgf_rule_s jsgf_rule_t;
+typedef struct jsgf_rhs_s jsgf_rhs_t;
+typedef struct jsgf_atom_s jsgf_atom_t;
+typedef struct jsgf_link_s jsgf_link_t;
+typedef struct jsgf_rule_stack_s jsgf_rule_stack_t;
 
 /**
  * Create a new JSGF grammar.
@@ -201,6 +205,71 @@ fsg_model_t *jsgf_read_string(const char *string, logmath_t * lmath, float32 lw)
 SPHINXBASE_EXPORT
 int jsgf_write_fsg(jsgf_t *grammar, jsgf_rule_t *rule, FILE *outfh);
 
+/* ------------ MOVED FROM jsgf_internal.h ---------- */
+#define jsgf_atom_is_rule(atom) ((atom)->name[0] == '<')
+
+void jsgf_add_link(jsgf_t *grammar, jsgf_atom_t *atom, int from, int to);
+jsgf_atom_t *jsgf_atom_new(char *name, float weight);
+jsgf_atom_t *jsgf_kleene_new(jsgf_t *jsgf, jsgf_atom_t *atom, int plus);
+jsgf_rule_t *jsgf_optional_new(jsgf_t *jsgf, jsgf_rhs_t *exp);
+jsgf_rule_t *jsgf_define_rule(jsgf_t *jsgf, char *name, jsgf_rhs_t *rhs, int is_public);
+jsgf_rule_t *jsgf_import_rule(jsgf_t *jsgf, char *name);
+
+int jsgf_atom_free(jsgf_atom_t *atom);
+int jsgf_rule_free(jsgf_rule_t *rule);
+jsgf_rule_t *jsgf_rule_retain(jsgf_rule_t *rule);
+
+/* ------------ MY FUNCTIONS ----------- */
+
+/**
+ * Creates a new atom with single tag.
+ *
+ */
+SPHINXBASE_EXPORT
+jsgf_atom_t *jsgf_atom_with_tag_new(char *name, float weight, char *tag);
+
+/**
+ * Creates a new rhs struct.
+ *
+ */
+SPHINXBASE_EXPORT
+jsgf_rhs_t *jsgf_rhs_new();
+
+/**
+ * Some getters and setters put to allow the single inclusion of jsgf.h.
+ *
+ */
+
+SPHINXBASE_EXPORT
+jsgf_rhs_t *jsgf_get_rule_rhs(jsgf_rule_t *rule);
+
+SPHINXBASE_EXPORT
+jsgf_rhs_t *jsgf_get_rhs_alt(jsgf_rhs_t *rhs);
+
+SPHINXBASE_EXPORT
+gnode_t *jsgf_get_rhs_atom(jsgf_rhs_t *rhs);
+
+SPHINXBASE_EXPORT
+char *jsgf_get_atom_name(jsgf_atom_t *atom);
+
+SPHINXBASE_EXPORT
+void jsgf_set_rule_rhs(jsgf_rule_t *rule, jsgf_rhs_t *rhs);
+
+SPHINXBASE_EXPORT
+void jsgf_set_rhs_alt(jsgf_rhs_t *rhs, jsgf_rhs_t *alt);
+
+SPHINXBASE_EXPORT
+void jsgf_set_rhs_atom(jsgf_rhs_t *rhs, gnode_t* gn);
+
+SPHINXBASE_EXPORT
+void jsgf_set_atom_name(jsgf_atom_t *atom, char *name);
+
+SPHINXBASE_EXPORT
+int jsgf_is_atom_rule(jsgf_atom_t *atom);
+
+SPHINXBASE_EXPORT
+int jsgf_rule_clean(jsgf_rule_t *rule);
+
 #ifdef __cplusplus
 }
 #endif

src/libsphinxbase/lm/fsg_model.c

@@ -8,24 +8,24 @@
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer. 
+ *    notice, this list of conditions and the following disclaimer.
  *
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in
  *    the documentation and/or other materials provided with the
  *    distribution.
  *
  *
- * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
- * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+ * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
+ * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
  * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * ====================================================================
@@ -111,7 +111,7 @@ nextline_str2words(FILE * fp, int32 * lineno,
 
 void
 fsg_model_trans_add(fsg_model_t * fsg,
-                    int32 from, int32 to, int32 logp, int32 wid)
+                    int32 from, int32 to, int32 logp, int32 wid, glist_t tags)
 {
     fsg_link_t *link;
     glist_t gl;
@@ -137,6 +137,12 @@ fsg_model_trans_add(fsg_model_t * fsg,
     link->logs2prob = logp;
     link->wid = wid;
 
+    gnode_t *gnt = tags;
+    if(gnt)
+    {
+        strncpy(link->tag,(char *)gnode_ptr(gnt),50);
+    }
+
     /* Add it to the list of transitions and update the hash table */
     gl = glist_add_ptr(gl, (void *) link);
     hash_table_replace_bkey(fsg->trans[from].trans,
@@ -208,7 +214,7 @@ fsg_model_null_trans_closure(fsg_model_t * fsg, glist_t nulls)
     E_INFO("Computing transitive closure for null transitions\n");
 
     /* If our caller didn't give us a list of null-transitions,
-       make such a list. Just loop through all the FSG states, 
+       make such a list. Just loop through all the FSG states,
        and all the null-transitions in that state (which are kept in
        their own hash table). */
     if (nulls == NULL) {
@@ -431,12 +437,12 @@ fsg_model_add_silence(fsg_model_t * fsg, char const *silword,
     n_trans = 0;
     if (state == -1) {
         for (src = 0; src < fsg->n_state; src++) {
-            fsg_model_trans_add(fsg, src, src, logsilp, silwid);
+            fsg_model_trans_add(fsg, src, src, logsilp, silwid, NULL);
             ++n_trans;
         }
     }
     else {
-        fsg_model_trans_add(fsg, state, state, logsilp, silwid);
+        fsg_model_trans_add(fsg, state, state, logsilp, silwid, NULL);
         ++n_trans;
     }
 
@@ -684,7 +690,7 @@ fsg_model_read(FILE * fp, logmath_t * lmath, float32 lw)
                 wid = lastwid;
                 ++lastwid;
             }
-            fsg_model_trans_add(fsg, i, j, tprob, wid);
+            fsg_model_trans_add(fsg, i, j, tprob, wid, NULL);
             ++n_trans;
         }
         else {