SphinxBase  0.6
jsgf.c
Go to the documentation of this file.
1 /* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /* ====================================================================
3  * Copyright (c) 2007 Carnegie Mellon University. All rights
4  * reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  *
10  * 1. Redistributions of source code must retain the above copyright
11  * notice, this list of conditions and the following disclaimer.
12  *
13  * 2. Redistributions in binary form must reproduce the above copyright
14  * notice, this list of conditions and the following disclaimer in
15  * the documentation and/or other materials provided with the
16  * distribution.
17  *
18  * This work was supported in part by funding from the Defense Advanced
19  * Research Projects Agency and the National Science Foundation of the
20  * United States of America, and the CMU Sphinx Speech Consortium.
21  *
22  * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
23  * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
26  * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33  *
34  * ====================================================================
35  *
36  */
37 
38 #include <string.h>
39 #include <assert.h>
40 
41 #include "sphinxbase/ckd_alloc.h"
42 #include "sphinxbase/strfuncs.h"
43 #include "sphinxbase/hash_table.h"
44 #include "sphinxbase/err.h"
45 
46 #include "jsgf_internal.h"
47 #include "jsgf_parser.h"
48 #include "jsgf_scanner.h"
49 
58 jsgf_atom_new(char *name, float weight)
59 {
60  jsgf_atom_t *atom;
61 
62  atom = ckd_calloc(1, sizeof(*atom));
63  atom->name = ckd_salloc(name);
64  atom->weight = weight;
65  return atom;
66 }
67 
68 int
69 jsgf_atom_free(jsgf_atom_t *atom)
70 {
71  if (atom == NULL)
72  return 0;
73  ckd_free(atom->name);
74  ckd_free(atom);
75  return 0;
76 }
77 
78 jsgf_t *
80 {
81  jsgf_t *grammar;
82 
83  grammar = ckd_calloc(1, sizeof(*grammar));
84  /* If this is an imported/subgrammar, then we will share a global
85  * namespace with the parent grammar. */
86  if (parent) {
87  grammar->rules = parent->rules;
88  grammar->imports = parent->imports;
89  grammar->searchpath = parent->searchpath;
90  grammar->parent = parent;
91  }
92  else {
93  char *jsgf_path;
94 
95  grammar->rules = hash_table_new(64, 0);
96  grammar->imports = hash_table_new(16, 0);
97 
98  /* Silvio Moioli: no getenv() in Windows CE */
99  #if !defined(_WIN32_WCE)
100  if ((jsgf_path = getenv("JSGF_PATH")) != NULL) {
101  char *word, *c;
102 
103  /* FIXME: This should be a function in libsphinxbase. */
104  /* FIXME: Also nextword() is totally useless... */
105  word = jsgf_path = ckd_salloc(jsgf_path);
106  while ((c = strchr(word, ':'))) {
107  *c = '\0';
108  grammar->searchpath = glist_add_ptr(grammar->searchpath, word);
109  word = c + 1;
110  }
111  grammar->searchpath = glist_add_ptr(grammar->searchpath, word);
112  grammar->searchpath = glist_reverse(grammar->searchpath);
113  }
114  else {
115  /* Default to current directory. */
116  grammar->searchpath = glist_add_ptr(grammar->searchpath, ckd_salloc("."));
117  }
118  #endif
119  }
120 
121  return grammar;
122 }
123 
124 void
126 {
127  /* FIXME: Probably should just use refcounting instead. */
128  if (jsgf->parent == NULL) {
129  hash_iter_t *itor;
130  gnode_t *gn;
131 
132  for (itor = hash_table_iter(jsgf->rules); itor;
133  itor = hash_table_iter_next(itor)) {
134  ckd_free((char *)itor->ent->key);
135  jsgf_rule_free((jsgf_rule_t *)itor->ent->val);
136  }
137  hash_table_free(jsgf->rules);
138  for (itor = hash_table_iter(jsgf->imports); itor;
139  itor = hash_table_iter_next(itor)) {
140  ckd_free((char *)itor->ent->key);
141  jsgf_grammar_free((jsgf_t *)itor->ent->val);
142  }
143  hash_table_free(jsgf->imports);
144  for (gn = jsgf->searchpath; gn; gn = gnode_next(gn))
145  ckd_free(gnode_ptr(gn));
146  glist_free(jsgf->searchpath);
147  for (gn = jsgf->links; gn; gn = gnode_next(gn))
148  ckd_free(gnode_ptr(gn));
149  glist_free(jsgf->links);
150  }
151  ckd_free(jsgf->name);
152  ckd_free(jsgf->version);
153  ckd_free(jsgf->charset);
154  ckd_free(jsgf->locale);
155  ckd_free(jsgf);
156 }
157 
158 static void
159 jsgf_rhs_free(jsgf_rhs_t *rhs)
160 {
161  gnode_t *gn;
162 
163  if (rhs == NULL)
164  return;
165 
166  jsgf_rhs_free(rhs->alt);
167  for (gn = rhs->atoms; gn; gn = gnode_next(gn))
168  jsgf_atom_free(gnode_ptr(gn));
169  glist_free(rhs->atoms);
170  ckd_free(rhs);
171 }
172 
173 jsgf_atom_t *
174 jsgf_kleene_new(jsgf_t *jsgf, jsgf_atom_t *atom, int plus)
175 {
176  jsgf_rule_t *rule;
177  jsgf_atom_t *rule_atom;
178  jsgf_rhs_t *rhs;
179 
180  /* Generate an "internal" rule of the form (<NULL> | <name> <g0006>) */
181  /* Or if plus is true, (<name> | <name> <g0006>) */
182  rhs = ckd_calloc(1, sizeof(*rhs));
183  if (plus)
184  rhs->atoms = glist_add_ptr(NULL, jsgf_atom_new(atom->name, 1.0));
185  else
186  rhs->atoms = glist_add_ptr(NULL, jsgf_atom_new("<NULL>", 1.0));
187  rule = jsgf_define_rule(jsgf, NULL, rhs, 0);
188  rule_atom = jsgf_atom_new(rule->name, 1.0);
189  rhs = ckd_calloc(1, sizeof(*rhs));
190  rhs->atoms = glist_add_ptr(NULL, rule_atom);
191  rhs->atoms = glist_add_ptr(rhs->atoms, atom);
192  rule->rhs->alt = rhs;
193 
194  return jsgf_atom_new(rule->name, 1.0);
195 }
196 
197 jsgf_rule_t *
198 jsgf_optional_new(jsgf_t *jsgf, jsgf_rhs_t *exp)
199 {
200  jsgf_rhs_t *rhs = ckd_calloc(1, sizeof(*rhs));
201  jsgf_atom_t *atom = jsgf_atom_new("<NULL>", 1.0);
202  rhs->alt = exp;
203  rhs->atoms = glist_add_ptr(NULL, atom);
204  return jsgf_define_rule(jsgf, NULL, rhs, 0);
205 }
206 
207 void
208 jsgf_add_link(jsgf_t *grammar, jsgf_atom_t *atom, int from, int to)
209 {
210  jsgf_link_t *link;
211 
212  link = ckd_calloc(1, sizeof(*link));
213  link->from = from;
214  link->to = to;
215  link->atom = atom;
216  grammar->links = glist_add_ptr(grammar->links, link);
217 }
218 
219 static char *
220 extract_grammar_name(char *rule_name)
221 {
222  char* dot_pos;
223  char* grammar_name = ckd_salloc(rule_name+1);
224  if ((dot_pos = strrchr(grammar_name + 1, '.')) == NULL) {
225  ckd_free(grammar_name);
226  return NULL;
227  }
228  *dot_pos='\0';
229  return grammar_name;
230 }
231 
232 char const *
234 {
235  return jsgf->name;
236 }
237 
238 static char *
239 jsgf_fullname(jsgf_t *jsgf, const char *name)
240 {
241  char *fullname;
242 
243  /* Check if it is already qualified */
244  if (strchr(name + 1, '.'))
245  return ckd_salloc(name);
246 
247  /* Skip leading < in name */
248  fullname = ckd_malloc(strlen(jsgf->name) + strlen(name) + 4);
249  sprintf(fullname, "<%s.%s", jsgf->name, name + 1);
250  return fullname;
251 }
252 
253 static char *
254 jsgf_fullname_from_rule(jsgf_rule_t *rule, const char *name)
255 {
256  char *fullname, *grammar_name;
257 
258  /* Check if it is already qualified */
259  if (strchr(name + 1, '.'))
260  return ckd_salloc(name);
261 
262  /* Skip leading < in name */
263  if ((grammar_name = extract_grammar_name(rule->name)) == NULL)
264  return ckd_salloc(name);
265  fullname = ckd_malloc(strlen(grammar_name) + strlen(name) + 4);
266  sprintf(fullname, "<%s.%s", grammar_name, name + 1);
267  ckd_free(grammar_name);
268 
269  return fullname;
270 }
271 
272 /* Extract as rulename everything after the secondlast dot, if existent.
273  * Because everything before the secondlast dot is the path-specification. */
274 static char *
275 importname2rulename(char *importname)
276 {
277  char *rulename = ckd_salloc(importname);
278  char *last_dotpos;
279  char *secondlast_dotpos;
280 
281  if ((last_dotpos = strrchr(rulename+1, '.')) != NULL) {
282  *last_dotpos='\0';
283  if ((secondlast_dotpos = strrchr(rulename+1, '.')) != NULL) {
284  *last_dotpos='.';
285  *secondlast_dotpos='<';
286  secondlast_dotpos = ckd_salloc(secondlast_dotpos);
287  ckd_free(rulename);
288  return secondlast_dotpos;
289  }
290  else {
291  *last_dotpos='.';
292  return rulename;
293  }
294  }
295  else {
296  return rulename;
297  }
298 }
299 
300 static int expand_rule(jsgf_t *grammar, jsgf_rule_t *rule);
301 static int
302 expand_rhs(jsgf_t *grammar, jsgf_rule_t *rule, jsgf_rhs_t *rhs)
303 {
304  gnode_t *gn;
305  int lastnode;
306 
307  /* Last node expanded in this sequence. */
308  lastnode = rule->entry;
309 
310  /* Iterate over atoms in rhs and generate links/nodes */
311  for (gn = rhs->atoms; gn; gn = gnode_next(gn)) {
312  jsgf_atom_t *atom = gnode_ptr(gn);
313  if (jsgf_atom_is_rule(atom)) {
314  jsgf_rule_t *subrule;
315  char *fullname;
316  gnode_t *subnode;
317  void *val;
318 
319  /* Special case for <NULL> and <VOID> pseudo-rules */
320  if (0 == strcmp(atom->name, "<NULL>")) {
321  /* Emit a NULL transition */
322  jsgf_add_link(grammar, atom,
323  lastnode, grammar->nstate);
324  lastnode = grammar->nstate;
325  ++grammar->nstate;
326  continue;
327  }
328  else if (0 == strcmp(atom->name, "<VOID>")) {
329  /* Make this entire RHS unspeakable */
330  return -1;
331  }
332 
333  fullname = jsgf_fullname_from_rule(rule, atom->name);
334  if (hash_table_lookup(grammar->rules, fullname, &val) == -1) {
335  E_ERROR("Undefined rule in RHS: %s\n", fullname);
336  ckd_free(fullname);
337  return -1;
338  }
339  ckd_free(fullname);
340  subrule = val;
341  /* Look for this in the stack of expanded rules */
342  for (subnode = grammar->rulestack; subnode; subnode = gnode_next(subnode))
343  if (gnode_ptr(subnode) == (void *)subrule)
344  break;
345  if (subnode != NULL) {
346  /* Allow right-recursion only. */
347  if (gnode_next(gn) != NULL) {
348  E_ERROR("Only right-recursion is permitted (in %s.%s)\n",
349  grammar->name, rule->name);
350  return -1;
351  }
352  /* Add a link back to the beginning of this rule instance */
353  E_INFO("Right recursion %s %d => %d\n", atom->name, lastnode, subrule->entry);
354  jsgf_add_link(grammar, atom, lastnode, subrule->entry);
355  }
356  else {
357  /* Expand the subrule */
358  if (expand_rule(grammar, subrule) == -1)
359  return -1;
360  /* Add a link into the subrule. */
361  jsgf_add_link(grammar, atom,
362  lastnode, subrule->entry);
363  lastnode = subrule->exit;
364  }
365  }
366  else {
367  /* Add a link for this token and create a new exit node. */
368  jsgf_add_link(grammar, atom,
369  lastnode, grammar->nstate);
370  lastnode = grammar->nstate;
371  ++grammar->nstate;
372  }
373  }
374 
375  return lastnode;
376 }
377 
378 static int
379 expand_rule(jsgf_t *grammar, jsgf_rule_t *rule)
380 {
381  jsgf_rhs_t *rhs;
382  float norm;
383 
384  /* Push this rule onto the stack */
385  grammar->rulestack = glist_add_ptr(grammar->rulestack, rule);
386 
387  /* Normalize weights for all alternatives exiting rule->entry */
388  norm = 0;
389  for (rhs = rule->rhs; rhs; rhs = rhs->alt) {
390  if (rhs->atoms) {
391  jsgf_atom_t *atom = gnode_ptr(rhs->atoms);
392  norm += atom->weight;
393  }
394  }
395 
396  rule->entry = grammar->nstate++;
397  rule->exit = grammar->nstate++;
398  if (norm == 0) norm = 1;
399  for (rhs = rule->rhs; rhs; rhs = rhs->alt) {
400  int lastnode;
401 
402  if (rhs->atoms) {
403  jsgf_atom_t *atom = gnode_ptr(rhs->atoms);
404  atom->weight /= norm;
405  }
406  lastnode = expand_rhs(grammar, rule, rhs);
407  if (lastnode == -1) {
408  return -1;
409  }
410  else {
411  jsgf_add_link(grammar, NULL, lastnode, rule->exit);
412  }
413  }
414 
415  /* Pop this rule from the rule stack */
416  grammar->rulestack = gnode_free(grammar->rulestack, NULL);
417  return rule->exit;
418 }
419 
422 {
423  return hash_table_iter(grammar->rules);
424 }
425 
426 jsgf_rule_t *
427 jsgf_get_rule(jsgf_t *grammar, char const *name)
428 {
429  void *val;
430 
431  if (hash_table_lookup(grammar->rules, name, &val) < 0)
432  return NULL;
433  return (jsgf_rule_t *)val;
434 }
435 
436 char const *
438 {
439  return rule->name;
440 }
441 
442 int
444 {
445  return rule->public;
446 }
447 
448 static fsg_model_t *
449 jsgf_build_fsg_internal(jsgf_t *grammar, jsgf_rule_t *rule,
450  logmath_t *lmath, float32 lw, int do_closure)
451 {
452  fsg_model_t *fsg;
453  glist_t nulls;
454  gnode_t *gn;
455 
456  /* Clear previous links */
457  for (gn = grammar->links; gn; gn = gnode_next(gn)) {
458  ckd_free(gnode_ptr(gn));
459  }
460  glist_free(grammar->links);
461  grammar->links = NULL;
462  rule->entry = rule->exit = 0;
463  grammar->nstate = 0;
464  expand_rule(grammar, rule);
465 
466  fsg = fsg_model_init(rule->name, lmath, lw, grammar->nstate);
467  fsg->start_state = rule->entry;
468  fsg->final_state = rule->exit;
469  grammar->links = glist_reverse(grammar->links);
470  for (gn = grammar->links; gn; gn = gnode_next(gn)) {
471  jsgf_link_t *link = gnode_ptr(gn);
472 
473  if (link->atom) {
474  if (jsgf_atom_is_rule(link->atom)) {
475  fsg_model_null_trans_add(fsg, link->from, link->to,
476  logmath_log(lmath, link->atom->weight));
477  }
478  else {
479  int wid = fsg_model_word_add(fsg, link->atom->name);
480  fsg_model_trans_add(fsg, link->from, link->to,
481  logmath_log(lmath, link->atom->weight), wid);
482  }
483  }
484  else {
485  fsg_model_null_trans_add(fsg, link->from, link->to, 0);
486  }
487  }
488  if (do_closure) {
489  nulls = fsg_model_null_trans_closure(fsg, NULL);
490  glist_free(nulls);
491  }
492 
493  return fsg;
494 }
495 
496 fsg_model_t *
498  logmath_t *lmath, float32 lw)
499 {
500  return jsgf_build_fsg_internal(grammar, rule, lmath, lw, TRUE);
501 }
502 
503 fsg_model_t *
505  logmath_t *lmath, float32 lw)
506 {
507  return jsgf_build_fsg_internal(grammar, rule, lmath, lw, FALSE);
508 }
509 
510 fsg_model_t *
511 jsgf_read_file(const char *file, logmath_t * lmath, float32 lw)
512 {
513  fsg_model_t *fsg;
514  jsgf_rule_t *rule;
515  jsgf_t *jsgf;
516  jsgf_rule_iter_t *itor;
517 
518  if ((jsgf = jsgf_parse_file(file, NULL)) == NULL) {
519  E_ERROR("Error parsing file: %s\n", file);
520  return NULL;
521  }
522 
523  rule = NULL;
524  for (itor = jsgf_rule_iter(jsgf); itor;
525  itor = jsgf_rule_iter_next(itor)) {
526  rule = jsgf_rule_iter_rule(itor);
527  if (jsgf_rule_public(rule)) {
528  jsgf_rule_iter_free(itor);
529  break;
530  }
531  }
532  if (rule == NULL) {
533  E_ERROR("No public rules found in %s\n", file);
534  return NULL;
535  }
536  fsg = jsgf_build_fsg(jsgf, rule, lmath, lw);
537  jsgf_grammar_free(jsgf);
538  return fsg;
539 }
540 
541 int
542 jsgf_write_fsg(jsgf_t *grammar, jsgf_rule_t *rule, FILE *outfh)
543 {
544  fsg_model_t *fsg;
545  logmath_t *lmath = logmath_init(1.0001, 0, 0);
546 
547  if ((fsg = jsgf_build_fsg_raw(grammar, rule, lmath, 1.0)) == NULL)
548  goto error_out;
549 
550  fsg_model_write(fsg, outfh);
551  logmath_free(lmath);
552  return 0;
553 
554 error_out:
555  logmath_free(lmath);
556  return -1;
557 }
558 jsgf_rule_t *
559 jsgf_define_rule(jsgf_t *jsgf, char *name, jsgf_rhs_t *rhs, int public)
560 {
561  jsgf_rule_t *rule;
562  void *val;
563 
564  if (name == NULL) {
565  name = ckd_malloc(strlen(jsgf->name) + 16);
566  sprintf(name, "<%s.g%05d>", jsgf->name, hash_table_inuse(jsgf->rules));
567  }
568  else {
569  char *newname;
570 
571  newname = jsgf_fullname(jsgf, name);
572  name = newname;
573  }
574 
575  rule = ckd_calloc(1, sizeof(*rule));
576  rule->refcnt = 1;
577  rule->name = ckd_salloc(name);
578  rule->rhs = rhs;
579  rule->public = public;
580 
581  E_INFO("Defined rule: %s%s\n",
582  rule->public ? "PUBLIC " : "",
583  rule->name);
584  val = hash_table_enter(jsgf->rules, name, rule);
585  if (val != (void *)rule) {
586  E_WARN("Multiply defined symbol: %s\n", name);
587  }
588  return rule;
589 }
590 
591 jsgf_rule_t *
592 jsgf_rule_retain(jsgf_rule_t *rule)
593 {
594  ++rule->refcnt;
595  return rule;
596 }
597 
598 int
599 jsgf_rule_free(jsgf_rule_t *rule)
600 {
601  if (rule == NULL)
602  return 0;
603  if (--rule->refcnt > 0)
604  return rule->refcnt;
605  jsgf_rhs_free(rule->rhs);
606  ckd_free(rule->name);
607  ckd_free(rule);
608  return 0;
609 }
610 
611 
612 /* FIXME: This should go in libsphinxutil */
613 static char *
614 path_list_search(glist_t paths, char *path)
615 {
616  gnode_t *gn;
617 
618  for (gn = paths; gn; gn = gnode_next(gn)) {
619  char *fullpath;
620  FILE *tmp;
621 
622  fullpath = string_join(gnode_ptr(gn), "/", path, NULL);
623  tmp = fopen(fullpath, "r");
624  if (tmp != NULL) {
625  fclose(tmp);
626  return fullpath;
627  }
628  else
629  ckd_free(fullpath);
630  }
631  return NULL;
632 }
633 
634 jsgf_rule_t *
635 jsgf_import_rule(jsgf_t *jsgf, char *name)
636 {
637  char *c, *path, *newpath;
638  size_t namelen, packlen;
639  void *val;
640  jsgf_t *imp;
641  int import_all;
642 
643  /* Trim the leading and trailing <> */
644  namelen = strlen(name);
645  path = ckd_malloc(namelen - 2 + 6); /* room for a trailing .gram */
646  strcpy(path, name + 1);
647  /* Split off the first part of the name */
648  c = strrchr(path, '.');
649  if (c == NULL) {
650  E_ERROR("Imported rule is not qualified: %s\n", name);
651  ckd_free(path);
652  return NULL;
653  }
654  packlen = c - path;
655  *c = '\0';
656 
657  /* Look for import foo.* */
658  import_all = (strlen(name) > 2 && 0 == strcmp(name + namelen - 3, ".*>"));
659 
660  /* Construct a filename. */
661  for (c = path; *c; ++c)
662  if (*c == '.') *c = '/';
663  strcat(path, ".gram");
664  newpath = path_list_search(jsgf->searchpath, path);
665  ckd_free(path);
666  if (newpath == NULL)
667  return NULL;
668 
669  path = newpath;
670  E_INFO("Importing %s from %s to %s\n", name, path, jsgf->name);
671 
672  /* FIXME: Also, we need to make sure that path is fully qualified
673  * here, by adding any prefixes from jsgf->name to it. */
674  /* See if we have parsed it already */
675  if (hash_table_lookup(jsgf->imports, path, &val) == 0) {
676  E_INFO("Already imported %s\n", path);
677  imp = val;
678  ckd_free(path);
679  }
680  else {
681  /* If not, parse it. */
682  imp = jsgf_parse_file(path, jsgf);
683  val = hash_table_enter(jsgf->imports, path, imp);
684  if (val != (void *)imp) {
685  E_WARN("Multiply imported file: %s\n", path);
686  }
687  }
688  if (imp != NULL) {
689  hash_iter_t *itor;
690  /* Look for public rules matching rulename. */
691  for (itor = hash_table_iter(imp->rules); itor;
692  itor = hash_table_iter_next(itor)) {
693  hash_entry_t *he = itor->ent;
694  jsgf_rule_t *rule = hash_entry_val(he);
695  int rule_matches;
696  char *rule_name = importname2rulename(name);
697 
698  if (import_all) {
699  /* Match package name (symbol table is shared) */
700  rule_matches = !strncmp(rule_name, rule->name, packlen + 1);
701  }
702  else {
703  /* Exact match */
704  rule_matches = !strcmp(rule_name, rule->name);
705  }
706  ckd_free(rule_name);
707  if (rule->public && rule_matches) {
708  void *val;
709  char *newname;
710 
711  /* Link this rule into the current namespace. */
712  c = strrchr(rule->name, '.');
713  assert(c != NULL);
714  newname = jsgf_fullname(jsgf, c);
715 
716  E_INFO("Imported %s\n", newname);
717  val = hash_table_enter(jsgf->rules, newname,
718  jsgf_rule_retain(rule));
719  if (val != (void *)rule) {
720  E_WARN("Multiply defined symbol: %s\n", newname);
721  }
722  if (!import_all) {
723  hash_table_iter_free(itor);
724  return rule;
725  }
726  }
727  }
728  }
729 
730  return NULL;
731 }
732 
733 jsgf_t *
734 jsgf_parse_file(const char *filename, jsgf_t *parent)
735 {
736  yyscan_t yyscanner;
737  jsgf_t *jsgf;
738  int yyrv;
739  FILE *in = NULL;
740 
741  yylex_init(&yyscanner);
742  if (filename == NULL) {
743  yyset_in(stdin, yyscanner);
744  }
745  else {
746  in = fopen(filename, "r");
747  if (in == NULL) {
748  E_ERROR_SYSTEM("Failed to open %s for parsing", filename);
749  return NULL;
750  }
751  yyset_in(in, yyscanner);
752  }
753 
754  jsgf = jsgf_grammar_new(parent);
755  yyrv = yyparse(yyscanner, jsgf);
756  if (yyrv != 0) {
757  E_ERROR("Failed to parse JSGF grammar from '%s'\n", filename ? filename : "(stdin)");
758  jsgf_grammar_free(jsgf);
759  yylex_destroy(yyscanner);
760  return NULL;
761  }
762  if (in)
763  fclose(in);
764  yylex_destroy(yyscanner);
765 
766  return jsgf;
767 }
jsgf_rule_t * jsgf_get_rule(jsgf_t *grammar, char const *name)
Get a rule by name from a grammar.
Definition: jsgf.c:427
int32 start_state
Must be in the range [0..n_state-1].
Definition: fsg_model.h:101
jsgf_t * jsgf_parse_file(const char *filename, jsgf_t *parent)
Parse a JSGF grammar from a file.
Definition: jsgf.c:734
Miscellaneous useful string functions.
Internal definitions for JSGF grammar compiler.
hash_entry_t * ent
Current entry in that table.
Definition: hash_table.h:170
SPHINXBASE_EXPORT int32 hash_table_lookup(hash_table_t *h, const char *key, void **val)
Look up a key in a hash table and optionally return the associated value.
Definition: hash_table.c:309
#define jsgf_rule_iter_next(itor)
Advance an iterator to the next rule in the grammar.
Definition: jsgf.h:112
int32 final_state
Must be in the range [0..n_state-1].
Definition: fsg_model.h:102
#define ckd_calloc(n, sz)
Macros to simplify the use of above functions.
Definition: ckd_alloc.h:248
#define E_INFO
Print logging information to standard error stream.
Definition: err.h:147
Sphinx's memory allocation/deallocation routines.
glist_t links
Generated FSG links.
Definition: jsgf_internal.h:82
SPHINXBASE_EXPORT void hash_table_iter_free(hash_iter_t *itor)
Delete an unfinished iterator.
Definition: hash_table.c:689
SPHINXBASE_EXPORT int logmath_log(logmath_t *lmath, float64 p)
Convert linear floating point number to integer log in base B.
Definition: logmath.c:447
A node in a generic list.
Definition: glist.h:100
int jsgf_rule_public(jsgf_rule_t *rule)
Test if a rule is public or not.
Definition: jsgf.c:443
SPHINXBASE_EXPORT hash_iter_t * hash_table_iter(hash_table_t *h)
Start iterating over key-value pairs in a hash table.
Definition: hash_table.c:653
SPHINXBASE_EXPORT int logmath_free(logmath_t *lmath)
Free a log table.
Definition: logmath.c:342
#define ckd_salloc(ptr)
Macro for ckd_salloc
Definition: ckd_alloc.h:264
fsg_model_t * jsgf_read_file(const char *file, logmath_t *lmath, float32 lw)
Read JSGF from file and return FSG object from it.
Definition: jsgf.c:511
int entry
Entry state for current instance of this rule.
Definition: jsgf_internal.h:92
#define hash_entry_val(e)
Access macros.
Definition: hash_table.h:175
SPHINXBASE_EXPORT hash_table_t * hash_table_new(int32 size, int32 casearg)
Allocate a new hash table for a given expected size.
Definition: hash_table.c:158
glist_t searchpath
List of directories to search for grammars.
Definition: jsgf_internal.h:78
char const * jsgf_grammar_name(jsgf_t *jsgf)
Get the grammar name from the file.
Definition: jsgf.c:233
SPHINXBASE_EXPORT void ckd_free(void *ptr)
Test and free a 1-D array.
Definition: ckd_alloc.c:241
SPHINXBASE_EXPORT glist_t glist_add_ptr(glist_t g, void *ptr)
Create and prepend a new list node, with the given user-defined data, at the HEAD of the given generi...
Definition: glist.c:74
#define E_WARN
Print warning information to standard error stream.
Definition: err.h:164
char * name
Grammar name.
Definition: jsgf_internal.h:73
jsgf_rhs_t * alt
Linked list of alternates.
Definition: jsgf_internal.h:98
SPHINXBASE_EXPORT logmath_t * logmath_init(float64 base, int shift, int use_table)
Initialize a log math computation table.
Definition: logmath.c:62
glist_t rulestack
Stack of currently expanded rules.
Definition: jsgf_internal.h:83
jsgf_rule_iter_t * jsgf_rule_iter(jsgf_t *grammar)
Get an iterator over all rules in a grammar.
Definition: jsgf.c:421
SPHINXBASE_EXPORT void hash_table_free(hash_table_t *h)
Free the specified hash table; the caller is responsible for freeing the key strings pointed to by th...
Definition: hash_table.c:695
int exit
Exit state for current instance of this rule.
Definition: jsgf_internal.h:93
int public
Is this rule marked 'public'?
Definition: jsgf_internal.h:89
int nstate
Number of generated states.
Definition: jsgf_internal.h:81
A note by ARCHAN at 20050510: Technically what we use is so-called "hash table with buckets" which is...
Definition: hash_table.h:149
char * charset
JSGF charset (default UTF-8)
Definition: jsgf_internal.h:71
SPHINXBASE_EXPORT glist_t glist_reverse(glist_t g)
Reverse the order of the given glist.
Definition: glist.c:169
jsgf_rhs_t * rhs
Expansion.
Definition: jsgf_internal.h:90
void jsgf_grammar_free(jsgf_t *jsgf)
Free a JSGF grammar.
Definition: jsgf.c:125
SPHINXBASE_EXPORT void glist_free(glist_t g)
Free the given generic list; user-defined data contained within is not automatically freed...
Definition: glist.c:133
SPHINXBASE_EXPORT gnode_t * gnode_free(gnode_t *gn, gnode_t *pred)
Free the given node, gn, of a glist, pred being its predecessor in the list.
Definition: glist.c:257
#define gnode_ptr(g)
Head of a list of gnodes.
Definition: glist.h:109
char * name
Rule name (NULL for an alternation/grouping)
Definition: jsgf_internal.h:88
Implementation of logging routines.
jsgf_t * jsgf_grammar_new(jsgf_t *parent)
Create a new JSGF grammar.
Definition: jsgf.c:79
int refcnt
Reference count.
Definition: jsgf_internal.h:87
SPHINXBASE_EXPORT void * hash_table_enter(hash_table_t *h, const char *key, void *val)
Try to add a new entry with given key and associated value to hash table h.
Definition: hash_table.c:508
#define jsgf_rule_iter_rule(itor)
Get the current rule in a rule iterator.
Definition: jsgf.h:117
fsg_model_t * jsgf_build_fsg_raw(jsgf_t *grammar, jsgf_rule_t *rule, logmath_t *lmath, float32 lw)
Build a Sphinx FSG object from a JSGF rule.
Definition: jsgf.c:504
char * version
JSGF version (from header)
Definition: jsgf_internal.h:70
SPHINXBASE_EXPORT hash_iter_t * hash_table_iter_next(hash_iter_t *itor)
Get the next key-value pair in iteration.
Definition: hash_table.c:663
#define ckd_malloc(sz)
Macro for ckd_malloc
Definition: ckd_alloc.h:253
glist_t atoms
Sequence of items.
Definition: jsgf_internal.h:97
SPHINXBASE_EXPORT char * string_join(const char *base,...)
Concatenate a NULL-terminated argument list of strings, returning a newly allocated string...
Definition: strfuncs.c:62
fsg_model_t * jsgf_build_fsg(jsgf_t *grammar, jsgf_rule_t *rule, logmath_t *lmath, float32 lw)
Build a Sphinx FSG object from a JSGF rule.
Definition: jsgf.c:497
jsgf_t * parent
Parent grammar (if this is an imported one)
Definition: jsgf_internal.h:77
char * name
Rule or token name.
#define E_ERROR
Print error message to standard error stream.
Definition: err.h:169
#define jsgf_rule_iter_free(itor)
Free a rule iterator (if the end hasn't been reached).
Definition: jsgf.h:122
hash_table_t * rules
Defined or imported rules in this grammar.
Definition: jsgf_internal.h:75
Hash table implementation.
char * locale
JSGF locale (default C)
Definition: jsgf_internal.h:72
void * val
Key-length; the key string does not have to be a C-style NULL terminated string; it can have arbitrar...
Definition: hash_table.h:155
Word level FSG definition.
Definition: fsg_model.h:91
int jsgf_write_fsg(jsgf_t *grammar, jsgf_rule_t *rule, FILE *outfh)
Convert a JSGF rule to Sphinx FSG text form.
Definition: jsgf.c:542
#define E_ERROR_SYSTEM
Print error text; Call perror("");.
Definition: err.h:142
float weight
Weight (default 1)
hash_table_t * imports
Pointers to imported grammars.
Definition: jsgf_internal.h:76
char const * jsgf_rule_name(jsgf_rule_t *rule)
Get the rule name from a rule.
Definition: jsgf.c:437