47 #include "jsgf_parser.h"
48 #include "jsgf_scanner.h"
58 jsgf_atom_new(
char *name,
float weight)
99 #if !defined(_WIN32_WCE)
100 if ((jsgf_path = getenv(
"JSGF_PATH")) != NULL) {
106 while ((c = strchr(word,
':'))) {
128 if (jsgf->
parent == NULL) {
144 for (gn = jsgf->
searchpath; gn; gn = gnode_next(gn))
147 for (gn = jsgf->
links; gn; gn = gnode_next(gn))
166 jsgf_rhs_free(rhs->
alt);
167 for (gn = rhs->
atoms; gn; gn = gnode_next(gn))
187 rule = jsgf_define_rule(jsgf, NULL, rhs, 0);
188 rule_atom = jsgf_atom_new(rule->
name, 1.0);
194 return jsgf_atom_new(rule->
name, 1.0);
204 return jsgf_define_rule(jsgf, NULL, rhs, 0);
220 extract_grammar_name(
char *rule_name)
224 if ((dot_pos = strrchr(grammar_name + 1,
'.')) == NULL) {
239 jsgf_fullname(
jsgf_t *jsgf,
const char *name)
244 if (strchr(name + 1,
'.'))
249 sprintf(fullname,
"<%s.%s", jsgf->
name, name + 1);
254 jsgf_fullname_from_rule(
jsgf_rule_t *rule,
const char *name)
256 char *fullname, *grammar_name;
259 if (strchr(name + 1,
'.'))
263 if ((grammar_name = extract_grammar_name(rule->
name)) == NULL)
265 fullname =
ckd_malloc(strlen(grammar_name) + strlen(name) + 4);
266 sprintf(fullname,
"<%s.%s", grammar_name, name + 1);
275 importname2rulename(
char *importname)
279 char *secondlast_dotpos;
281 if ((last_dotpos = strrchr(rulename+1,
'.')) != NULL) {
283 if ((secondlast_dotpos = strrchr(rulename+1,
'.')) != NULL) {
285 *secondlast_dotpos=
'<';
286 secondlast_dotpos =
ckd_salloc(secondlast_dotpos);
288 return secondlast_dotpos;
308 lastnode = rule->
entry;
311 for (gn = rhs->
atoms; gn; gn = gnode_next(gn)) {
313 if (jsgf_atom_is_rule(atom)) {
320 if (0 == strcmp(atom->
name,
"<NULL>")) {
322 jsgf_add_link(grammar, atom,
323 lastnode, grammar->
nstate);
324 lastnode = grammar->
nstate;
328 else if (0 == strcmp(atom->
name,
"<VOID>")) {
333 fullname = jsgf_fullname_from_rule(rule, atom->
name);
335 E_ERROR(
"Undefined rule in RHS: %s\n", fullname);
342 for (subnode = grammar->
rulestack; subnode; subnode = gnode_next(subnode))
343 if (
gnode_ptr(subnode) == (
void *)subrule)
345 if (subnode != NULL) {
347 if (gnode_next(gn) != NULL) {
348 E_ERROR(
"Only right-recursion is permitted (in %s.%s)\n",
353 E_INFO(
"Right recursion %s %d => %d\n", atom->
name, lastnode, subrule->
entry);
354 jsgf_add_link(grammar, atom, lastnode, subrule->
entry);
358 if (expand_rule(grammar, subrule) == -1)
361 jsgf_add_link(grammar, atom,
362 lastnode, subrule->
entry);
363 lastnode = subrule->
exit;
368 jsgf_add_link(grammar, atom,
369 lastnode, grammar->
nstate);
370 lastnode = grammar->
nstate;
389 for (rhs = rule->
rhs; rhs; rhs = rhs->
alt) {
398 if (norm == 0) norm = 1;
399 for (rhs = rule->
rhs; rhs; rhs = rhs->
alt) {
406 lastnode = expand_rhs(grammar, rule, rhs);
407 if (lastnode == -1) {
411 jsgf_add_link(grammar, NULL, lastnode, rule->
exit);
450 logmath_t *lmath, float32 lw,
int do_closure)
457 for (gn = grammar->
links; gn; gn = gnode_next(gn)) {
461 grammar->
links = NULL;
464 expand_rule(grammar, rule);
466 fsg = fsg_model_init(rule->
name, lmath, lw, grammar->
nstate);
470 for (gn = grammar->
links; gn; gn = gnode_next(gn)) {
474 if (jsgf_atom_is_rule(link->
atom)) {
475 fsg_model_null_trans_add(fsg, link->
from, link->
to,
479 int wid = fsg_model_word_add(fsg, link->
atom->
name);
480 fsg_model_trans_add(fsg, link->
from, link->
to,
485 fsg_model_null_trans_add(fsg, link->
from, link->
to, 0);
489 nulls = fsg_model_null_trans_closure(fsg, NULL);
500 return jsgf_build_fsg_internal(grammar, rule, lmath, lw, TRUE);
507 return jsgf_build_fsg_internal(grammar, rule, lmath, lw, FALSE);
519 E_ERROR(
"Error parsing file: %s\n", file);
533 E_ERROR(
"No public rules found in %s\n", file);
550 fsg_model_write(fsg, outfh);
566 sprintf(name,
"<%s.g%05d>", jsgf->
name, hash_table_inuse(jsgf->
rules));
571 newname = jsgf_fullname(jsgf, name);
581 E_INFO(
"Defined rule: %s%s\n",
582 rule->
public ?
"PUBLIC " :
"",
585 if (val != (
void *)rule) {
586 E_WARN(
"Multiply defined symbol: %s\n", name);
605 jsgf_rhs_free(rule->
rhs);
614 path_list_search(
glist_t paths,
char *path)
618 for (gn = paths; gn; gn = gnode_next(gn)) {
623 tmp = fopen(fullpath,
"r");
635 jsgf_import_rule(
jsgf_t *jsgf,
char *name)
637 char *c, *path, *newpath;
638 size_t namelen, packlen;
644 namelen = strlen(name);
646 strcpy(path, name + 1);
648 c = strrchr(path,
'.');
650 E_ERROR(
"Imported rule is not qualified: %s\n", name);
658 import_all = (strlen(name) > 2 && 0 == strcmp(name + namelen - 3,
".*>"));
661 for (c = path; *c; ++c)
662 if (*c ==
'.') *c =
'/';
663 strcat(path,
".gram");
664 newpath = path_list_search(jsgf->
searchpath, path);
670 E_INFO(
"Importing %s from %s to %s\n", name, path, jsgf->
name);
676 E_INFO(
"Already imported %s\n", path);
684 if (val != (
void *)imp) {
685 E_WARN(
"Multiply imported file: %s\n", path);
696 char *rule_name = importname2rulename(name);
700 rule_matches = !strncmp(rule_name, rule->
name, packlen + 1);
704 rule_matches = !strcmp(rule_name, rule->
name);
707 if (rule->
public && rule_matches) {
712 c = strrchr(rule->
name,
'.');
714 newname = jsgf_fullname(jsgf, c);
716 E_INFO(
"Imported %s\n", newname);
718 jsgf_rule_retain(rule));
719 if (val != (
void *)rule) {
720 E_WARN(
"Multiply defined symbol: %s\n", newname);
741 yylex_init(&yyscanner);
742 if (filename == NULL) {
743 yyset_in(stdin, yyscanner);
746 in = fopen(filename,
"r");
751 yyset_in(in, yyscanner);
755 yyrv = yyparse(yyscanner, jsgf);
757 E_ERROR(
"Failed to parse JSGF grammar from '%s'\n", filename ? filename :
"(stdin)");
759 yylex_destroy(yyscanner);
764 yylex_destroy(yyscanner);
jsgf_rule_t * jsgf_get_rule(jsgf_t *grammar, char const *name)
Get a rule by name from a grammar.
int32 start_state
Must be in the range [0..n_state-1].
jsgf_t * jsgf_parse_file(const char *filename, jsgf_t *parent)
Parse a JSGF grammar from a file.
Miscellaneous useful string functions.
Internal definitions for JSGF grammar compiler.
hash_entry_t * ent
Current entry in that table.
SPHINXBASE_EXPORT int32 hash_table_lookup(hash_table_t *h, const char *key, void **val)
Look up a key in a hash table and optionally return the associated value.
#define jsgf_rule_iter_next(itor)
Advance an iterator to the next rule in the grammar.
int32 final_state
Must be in the range [0..n_state-1].
#define ckd_calloc(n, sz)
Macros to simplify the use of above functions.
#define E_INFO
Print logging information to standard error stream.
Sphinx's memory allocation/deallocation routines.
glist_t links
Generated FSG links.
SPHINXBASE_EXPORT void hash_table_iter_free(hash_iter_t *itor)
Delete an unfinished iterator.
SPHINXBASE_EXPORT int logmath_log(logmath_t *lmath, float64 p)
Convert linear floating point number to integer log in base B.
A node in a generic list.
int jsgf_rule_public(jsgf_rule_t *rule)
Test if a rule is public or not.
SPHINXBASE_EXPORT hash_iter_t * hash_table_iter(hash_table_t *h)
Start iterating over key-value pairs in a hash table.
SPHINXBASE_EXPORT int logmath_free(logmath_t *lmath)
Free a log table.
#define ckd_salloc(ptr)
Macro for ckd_salloc
fsg_model_t * jsgf_read_file(const char *file, logmath_t *lmath, float32 lw)
Read JSGF from file and return FSG object from it.
int entry
Entry state for current instance of this rule.
#define hash_entry_val(e)
Access macros.
SPHINXBASE_EXPORT hash_table_t * hash_table_new(int32 size, int32 casearg)
Allocate a new hash table for a given expected size.
glist_t searchpath
List of directories to search for grammars.
char const * jsgf_grammar_name(jsgf_t *jsgf)
Get the grammar name from the file.
SPHINXBASE_EXPORT void ckd_free(void *ptr)
Test and free a 1-D array.
SPHINXBASE_EXPORT glist_t glist_add_ptr(glist_t g, void *ptr)
Create and prepend a new list node, with the given user-defined data, at the HEAD of the given generi...
#define E_WARN
Print warning information to standard error stream.
jsgf_rhs_t * alt
Linked list of alternates.
SPHINXBASE_EXPORT logmath_t * logmath_init(float64 base, int shift, int use_table)
Initialize a log math computation table.
glist_t rulestack
Stack of currently expanded rules.
jsgf_rule_iter_t * jsgf_rule_iter(jsgf_t *grammar)
Get an iterator over all rules in a grammar.
SPHINXBASE_EXPORT void hash_table_free(hash_table_t *h)
Free the specified hash table; the caller is responsible for freeing the key strings pointed to by th...
int exit
Exit state for current instance of this rule.
int public
Is this rule marked 'public'?
int nstate
Number of generated states.
A note by ARCHAN at 20050510: Technically what we use is so-called "hash table with buckets" which is...
char * charset
JSGF charset (default UTF-8)
SPHINXBASE_EXPORT glist_t glist_reverse(glist_t g)
Reverse the order of the given glist.
jsgf_rhs_t * rhs
Expansion.
void jsgf_grammar_free(jsgf_t *jsgf)
Free a JSGF grammar.
SPHINXBASE_EXPORT void glist_free(glist_t g)
Free the given generic list; user-defined data contained within is not automatically freed...
SPHINXBASE_EXPORT gnode_t * gnode_free(gnode_t *gn, gnode_t *pred)
Free the given node, gn, of a glist, pred being its predecessor in the list.
#define gnode_ptr(g)
Head of a list of gnodes.
char * name
Rule name (NULL for an alternation/grouping)
Implementation of logging routines.
jsgf_t * jsgf_grammar_new(jsgf_t *parent)
Create a new JSGF grammar.
int refcnt
Reference count.
SPHINXBASE_EXPORT void * hash_table_enter(hash_table_t *h, const char *key, void *val)
Try to add a new entry with given key and associated value to hash table h.
jsgf_atom_t * atom
Name, tags, weight.
#define jsgf_rule_iter_rule(itor)
Get the current rule in a rule iterator.
fsg_model_t * jsgf_build_fsg_raw(jsgf_t *grammar, jsgf_rule_t *rule, logmath_t *lmath, float32 lw)
Build a Sphinx FSG object from a JSGF rule.
char * version
JSGF version (from header)
SPHINXBASE_EXPORT hash_iter_t * hash_table_iter_next(hash_iter_t *itor)
Get the next key-value pair in iteration.
#define ckd_malloc(sz)
Macro for ckd_malloc
glist_t atoms
Sequence of items.
SPHINXBASE_EXPORT char * string_join(const char *base,...)
Concatenate a NULL-terminated argument list of strings, returning a newly allocated string...
fsg_model_t * jsgf_build_fsg(jsgf_t *grammar, jsgf_rule_t *rule, logmath_t *lmath, float32 lw)
Build a Sphinx FSG object from a JSGF rule.
jsgf_t * parent
Parent grammar (if this is an imported one)
char * name
Rule or token name.
#define E_ERROR
Print error message to standard error stream.
#define jsgf_rule_iter_free(itor)
Free a rule iterator (if the end hasn't been reached).
hash_table_t * rules
Defined or imported rules in this grammar.
Hash table implementation.
char * locale
JSGF locale (default C)
void * val
Key-length; the key string does not have to be a C-style NULL terminated string; it can have arbitrar...
Word level FSG definition.
int jsgf_write_fsg(jsgf_t *grammar, jsgf_rule_t *rule, FILE *outfh)
Convert a JSGF rule to Sphinx FSG text form.
#define E_ERROR_SYSTEM
Print error text; Call perror("");.
float weight
Weight (default 1)
hash_table_t * imports
Pointers to imported grammars.
char const * jsgf_rule_name(jsgf_rule_t *rule)
Get the rule name from a rule.