From 2b36dd4726fea2e59260c05378864faa2ade53ce Mon Sep 17 00:00:00 2001 From: ulatekh Date: Mon, 10 Sep 2018 18:36:21 -0700 Subject: [PATCH] Fixed jsgf_build_fsg_internal() to handle right-recursion in the presence of alternates. --- src/libsphinxbase/lm/jsgf.c | 149 ++++++++++++++++++++++++++- src/libsphinxbase/lm/jsgf_internal.h | 4 +- 2 files changed, 149 insertions(+), 4 deletions(-) diff --git a/src/libsphinxbase/lm/jsgf.c b/src/libsphinxbase/lm/jsgf.c index ab123734..3f8e3e99 100644 --- a/src/libsphinxbase/lm/jsgf.c +++ b/src/libsphinxbase/lm/jsgf.c @@ -58,6 +58,7 @@ extern int yyparse(void *scanner, jsgf_t * jsgf); * into Sphinx finite-state grammars. **/ +static void detect_recursion_rule(jsgf_t * grammar, jsgf_rule_t * rule); static int expand_rule(jsgf_t * grammar, jsgf_rule_t * rule, int rule_entry, int rule_exit); @@ -283,6 +284,100 @@ importname2rulename(char *importname) #define NO_NODE -1 #define RECURSIVE_NODE -2 +/** + * + * Detect recursion; label each RHS as to whether it's right-recursive. + * Just a cut-down version of expand_rhs() and expand_rule(). + */ +static void +detect_recursion_rhs(jsgf_t * grammar, jsgf_rule_t * rule, jsgf_rhs_t * rhs) +{ + gnode_t *gn; + + /* Iterate over atoms in rhs */ + for (gn = rhs->atoms; gn; gn = gnode_next(gn)) { + jsgf_atom_t *atom = (jsgf_atom_t *) gnode_ptr(gn); + + if (jsgf_atom_is_rule(atom)) { + jsgf_rule_t *subrule; + char *fullname; + int32 subrule_lookup_result; + gnode_t *subnode; + jsgf_rule_stack_t *rule_stack_entry = NULL; + + if (0 == strcmp(atom->name, "")) { + continue; + } + else if (0 == strcmp(atom->name, "")) { + /* This entire RHS is unspeakable */ + return; + } + + fullname = jsgf_fullname_from_rule(rule, atom->name); + subrule_lookup_result = hash_table_lookup + (grammar->rules, fullname, (void **) &subrule); + ckd_free(fullname); + if (subrule_lookup_result == -1) { + E_ERROR("Undefined rule in RHS: %s\n", fullname); + return; + } + + /* Look for this subrule in the stack of expanded rules */ + for (subnode = grammar->rulestack; subnode; + subnode = gnode_next(subnode)) { + rule_stack_entry = + (jsgf_rule_stack_t *) gnode_ptr(subnode); + if (rule_stack_entry->rule == subrule) + break; + } + + if (subnode != NULL) { + /* Allow right-recursion only. */ + if (gnode_next(gn) != NULL) { + E_ERROR + ("Only right-recursion is permitted (in %s.%s)\n", + grammar->name, rule->name); + return; + } + + /* Remember this RHS is right-recursive. */ + rhs->is_recursive = 1; + } + + /* If right-recursion has already been checked on this rule, + there's no need to check it again. */ + else if (!subrule->ck_recursive) { + /* Expand the subrule */ + detect_recursion_rule(grammar, subrule); + } + } + } +} + +static void +detect_recursion_rule(jsgf_t * grammar, jsgf_rule_t * rule) +{ + jsgf_rule_stack_t *rule_stack_entry; + jsgf_rhs_t *rhs; + + /* Push this rule onto the stack */ + rule_stack_entry = + (jsgf_rule_stack_t *) ckd_calloc(1, sizeof(jsgf_rule_stack_t)); + rule_stack_entry->rule = rule; + rule_stack_entry->entry = NO_NODE /* unused */; + grammar->rulestack = glist_add_ptr(grammar->rulestack, + rule_stack_entry); + + rule->ck_recursive = 1; + for (rhs = rule->rhs; rhs; rhs = rhs->alt) { + detect_recursion_rhs(grammar, rule, rhs); + } + + /* Pop this rule from the rule stack */ + ckd_free(gnode_ptr(grammar->rulestack)); + grammar->rulestack = gnode_free(grammar->rulestack, NULL); +} + /** * * Expand a right-hand-side of a rule (i.e. a single alternate). @@ -415,6 +510,7 @@ expand_rule(jsgf_t * grammar, jsgf_rule_t * rule, int rule_entry, int rule_exit) { jsgf_rule_stack_t *rule_stack_entry; + int multiple_alternates; jsgf_rhs_t *rhs; /* Push this rule onto the stack */ @@ -425,18 +521,32 @@ expand_rule(jsgf_t * grammar, jsgf_rule_t * rule, int rule_entry, grammar->rulestack = glist_add_ptr(grammar->rulestack, rule_stack_entry); + /* Remember whether this rule has more than one alternate. + This determines whether a null-transition has to be added + in order to to separate right-recursive sub-rules from + its sibling alternates. */ + multiple_alternates = (rule->rhs != NULL && rule->rhs->alt != NULL); + + /* Deal with all non-right-recursive rules. (They will need + the exit-node generated here.) */ for (rhs = rule->rhs; rhs; rhs = rhs->alt) { int lastnode; + /* If this rule is right-recursive, deal with it in the + next pass. */ + if (rhs->is_recursive && multiple_alternates) { + continue; + } + lastnode = expand_rhs(grammar, rule, rhs, rule_entry, rule_exit); if (lastnode == NO_NODE) { return NO_NODE; } else if (lastnode == RECURSIVE_NODE) { - /* The rhs ended with right-recursion, i.e. a transition to - an earlier state. Nothing needs to happen at this level. */ - ; + /* Shouldn't happen in this pass. */ + E_ERROR("Internal error"); + return NO_NODE; } else if (rule_exit == NO_NODE) { /* If this rule doesn't have an exit state yet, use the exit @@ -452,6 +562,36 @@ expand_rule(jsgf_t * grammar, jsgf_rule_t * rule, int rule_entry, rule_exit = rule_entry; } + /* Now deal with all the right-recursive rules. */ + for (rhs = rule->rhs; rhs; rhs = rhs->alt) { + int this_rule_entry, lastnode; + + /* If this rule is right-recursive, insert a null-transition to + give the right-recursion a unique destination, i.e. so that + it doesn't interfere with any of its sibling alternates. */ + this_rule_entry = rule_entry; + if (!rhs->is_recursive || !multiple_alternates) { + continue; + } + jsgf_add_link(grammar, NULL, + rule_entry, grammar->nstate); + this_rule_entry = grammar->nstate; + ++grammar->nstate; + rule_stack_entry->entry = this_rule_entry; + + lastnode = expand_rhs(grammar, rule, rhs, this_rule_entry, rule_exit); + + if (lastnode != RECURSIVE_NODE) { + /* Shouldn't happen in this pass. */ + E_ERROR("Internal error"); + return NO_NODE; + } + + /* Allow the right-recursion to exit. */ + jsgf_add_link(grammar, NULL, + this_rule_entry, rule_exit); + } + /* Pop this rule from the rule stack */ ckd_free(gnode_ptr(grammar->rulestack)); grammar->rulestack = gnode_free(grammar->rulestack, NULL); @@ -541,6 +681,9 @@ jsgf_build_fsg_internal(jsgf_t * grammar, jsgf_rule_t * rule, grammar->links = NULL; grammar->nstate = 0; + /* Determine which rules are right-recursive. */ + detect_recursion_rule(grammar, rule); + /* Create the top-level entry state, and expand the top-level rule. */ rule_entry = grammar->nstate++; diff --git a/src/libsphinxbase/lm/jsgf_internal.h b/src/libsphinxbase/lm/jsgf_internal.h index a5cbc983..69c8efe5 100644 --- a/src/libsphinxbase/lm/jsgf_internal.h +++ b/src/libsphinxbase/lm/jsgf_internal.h @@ -85,7 +85,7 @@ struct jsgf_s { /* Scratch variables for FSG conversion. */ int nstate; /**< Number of generated states. */ - glist_t links; /**< Generated FSG links. */ + glist_t links; /**< Generated FSG links. */ glist_t rulestack; /**< Stack of currently expanded rules. */ }; @@ -99,10 +99,12 @@ struct jsgf_rule_s { int refcnt; /**< Reference count. */ char *name; /**< Rule name (NULL for an alternation/grouping) */ int is_public; /**< Is this rule marked 'public'? */ + int ck_recursive;/**< Has right-recursion been checked on this rule already? */ jsgf_rhs_t *rhs; /**< Expansion */ }; struct jsgf_rhs_s { + int is_recursive;/**< Does this RHS recurse to its containing rule? */ glist_t atoms; /**< Sequence of items */ jsgf_rhs_t *alt; /**< Linked list of alternates */ };