Initial commit: FunASR Speech Recognition Toolkit

Add complete FunASR codebase including models, runtime, and documentation.

runtime/tools/fst/generate_lexicon.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+
+import sys
+
+# sys.argv[1]: lm dict
+# sys.argv[2]: lexicon file
+# sys.argv[3]: lexicon file for corpus.dict
+
+lex_dict = {}
+with open(sys.argv[2], "r", encoding="utf8") as fin:
+    for line in fin:
+        words = line.strip().split("\t")
+        if len(words) != 2:
+            continue
+        lex_dict[words[0]] = words[1]
+
+with open(sys.argv[1], "r", encoding="utf8") as fin, open(
+    sys.argv[3], "w", encoding="utf8"
+) as fout:
+    for line in fin:
+        word = line.strip()
+        if word == "<s>" or word == "</s>":
+            continue
+        word_lex = ""
+        if word in lex_dict:
+            word_lex = lex_dict[word]
+        else:
+            for i in range(len(word)):
+                if word[i] in lex_dict:
+                    word_lex += " " + lex_dict[word[i]]
+                else:
+                    word_lex += " <unk>"
+
+        fout.write("{}\t{}\n".format(word, word_lex.strip()))