Initial commit: FunASR Speech Recognition Toolkit

Add complete FunASR codebase including models, runtime, and documentation.

runtime/onnxruntime/third_party/kaldi/lm/Makefile

@@ -0,0 +1,25 @@
+EXTRA_CXXFLAGS = -Wno-sign-compare
+
+all:
+
+include ../kaldi.mk
+
+TESTFILES = arpa-file-parser-test arpa-lm-compiler-test
+
+OBJFILES = arpa-file-parser.o arpa-lm-compiler.o const-arpa-lm.o \
+	   kaldi-rnnlm.o mikolov-rnnlm-lib.o
+
+ifdef KENLM_ROOT
+TESTFILES += kenlm-test
+OBJFILES += kenlm.o
+EXTRA_CXXFLAGS += $(KENLM_CXXFLAGS)
+EXTRA_LDFLAGS += $(KENLM_LDFLAGS)
+EXTRA_LDLIBS += $(KENLM_LDLIBS)
+endif
+
+LIBNAME = kaldi-lm
+
+ADDLIBS = ../fstext/kaldi-fstext.a ../util/kaldi-util.a \
+          ../matrix/kaldi-matrix.a ../base/kaldi-base.a
+
+include ../makefiles/default_rules.mk

runtime/onnxruntime/third_party/kaldi/lm/README

@@ -0,0 +1,108 @@
+#
+# README
+#
+# Language model & lexicon examples
+# using command-line executables in lm/
+
+# To print and display FSTs,
+# make sure you have OpenFst binaries in your PATH, for example:
+#   export PATH=$PATH:~/Sources/UBM-ASR/branches/clean/openfst-1.2/bin
+
+# If you have X installed in your local machine, you can display
+# FSTs from merlin by ssh'ing with X forwarding, for example:
+#    ssh -X qboulianne@merlin
+
+# The following commands and examples assume that your are
+# in working directory UBM-ASR/branches/clean/src/lm
+
+#-------------------------------------------
+# Language model FST (G)
+
+# The command-line utility for
+# creating a language model FST from an arpa file is
+# "arpa2fst".
+
+# A summary of options and usage can be displayed with:
+./arpa2fst --help
+
+# Read an arpa file to produce an FST with symbol tables:
+./arpa2fst < input.arpa > grammar.fst
+
+# Print it or display it:
+fstprint grammar.fst
+fstdraw grammar.fst | dotty -
+
+# Note that arpa2fst will create a word symbol table
+# from all the words are in the ARPA file.
+# You can save this symbol table in text format
+# for examination or later reuse.
+fstprint --save_isymbols=grammar.syms grammar.fst > /dev/null
+
+#----------------------------------------------
+# Lexicon (L)
+
+# The command-line utility for
+# creating a lexicon FST from a text file is
+# "lex2fst".
+
+# A summary of options and usage can be displayed with:
+./lex2fst --help
+
+# Read a lexicon file (containing pronunciation probabilities)
+# and produce an FST with symbol tables.
+# By default it will have disambiguation markers,
+#   optional silence between words,
+#   and FST weights will be -log(prob).
+./lex2fst < prob_input.lex > lexicon.fst
+
+# Print it or display it
+fstprint lexicon.fst
+fstdraw lexicon.fst | dotty -
+
+# To produce one without markers (and also smaller):
+./lex2fst --nodisamb < prob_input.lex > lexicon_nomarkers.fst
+
+#---------------------------------
+# Combining lexicon and grammar
+
+# lgrecipe.cc is an example recipe for building det(LoG)
+# in C++ using calls to functions in lm/kaldi-lm.a
+# Here we use a large lexicon and language model from last year's.
+# First get input files:
+export MDIR=/homes/eva/q/qgoel/englishModels
+cp $MDIR/lm_callhome_gigaword_switchboard_web.hd.dct largelexicon.dct
+gunzip -c $MDIR/lm_callhome_gigaword_switchboard_web.3gram.arpa.gz > largelm.arpa
+
+# Set memory limits (by default limited to 400 MB). Make it 4 GB.
+ulimit -m 4000000 -v 8000000
+./lgrecipe largelexicon.dct largelm.arpa detlg.fst
+
+# Check its size: should be close to 13 M arcs and 8 M nodes
+fstinfo detlg.fst | head -15
+
+#---------------------------------------
+# Expanding nested grammars
+
+# replace-example is a use-case example that creates
+# small grammars from text files. These grammars
+# refer to other grammars using non-terminal symbols.
+# The result is the fully expanded grammar FST,
+# where each non-terminal has been expanded entirely
+# to non-terminals.
+cd examples
+
+# Example 1 : create the expanded grammar
+../replace-example input3.txt CREATURE.txt > input3.fst
+
+# and generate a random sentence from the expanded grammar
+fstrandgen input3.fst | fstrmepsilon | fstprint | cut -f 4
+
+# Example 2 : create the expanded grammar
+../replace-example input4.txt DAYOFMONTH.txt MONTH.txt YEAR.txt YEARDATE.txt > input4.fst
+
+# and generate random sentence
+fstrandgen input4.fst | fstproject --project_output | fstrmepsilon | fstprint | cut -f 4
+
+#--------------------------------------------
+# TODO: Adapt README.testfiles as an example of how to create
+#  an arpa language model and evaluate its score / perplexity.

runtime/onnxruntime/third_party/kaldi/lm/arpa-file-parser-test.cc

@@ -0,0 +1,373 @@
+// lm/arpa-file-parser-test.cc
+
+// Copyright 2016  Smart Action Company LLC (kkm)
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//  http://www.apache.org/licenses/LICENSE-2.0
+
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file arpa-file-parser-test.cc
+ * @brief Unit tests for language model code.
+ */
+
+#include <iomanip>
+#include <iostream>
+#include <string>
+#include <sstream>
+#include <vector>
+
+#include "base/kaldi-common.h"
+#include "fst/fstlib.h"
+#include "lm/arpa-file-parser.h"
+
+namespace kaldi {
+namespace {
+
+const int kMaxOrder = 3;
+
+struct NGramTestData {
+  int32 line_number;
+  float logprob;
+  int32 words[kMaxOrder];
+  float backoff;
+};
+
+std::ostream& operator<<(std::ostream &os, const NGramTestData &data) {
+  std::ios::fmtflags saved_state(os.flags());
+  os << std::fixed << std::setprecision(6);
+
+  os << data.logprob << ' ';
+  for (int i = 0; i < kMaxOrder; ++i) os << data.words[i] << ' ';
+  os << data.backoff << " // Line " << data.line_number;
+
+  os.flags(saved_state);
+  return os;
+}
+
+// This does not own the array pointer, and uset to simplify passing expected
+// result to TestableArpaFileParser::Verify.
+template <class T>
+struct CountedArray {
+  template <size_t N>
+  CountedArray(T(&array)[N]) : array(array), count(N) { }
+  const T *array;
+  const size_t count;
+};
+
+template <class T, size_t N>
+inline CountedArray<T> MakeCountedArray(T(&array)[N]) {
+  return CountedArray<T>(array);
+}
+
+class TestableArpaFileParser : public ArpaFileParser {
+ public:
+  TestableArpaFileParser(const ArpaParseOptions &options,
+                         fst::SymbolTable *symbols)
+      : ArpaFileParser(options, symbols),
+        header_available_(false),
+        read_complete_(false),
+        last_order_(0) { }
+  void Validate(CountedArray<int32> counts, CountedArray<NGramTestData> ngrams);
+
+ private:
+  // ArpaFileParser overrides.
+  virtual void HeaderAvailable();
+  virtual void ConsumeNGram(const NGram& ngram);
+  virtual void ReadComplete();
+
+  bool header_available_;
+  bool read_complete_;
+  int32 last_order_;
+  std::vector<NGramTestData> ngrams_;
+};
+
+void TestableArpaFileParser::HeaderAvailable() {
+  KALDI_ASSERT(!header_available_);
+  KALDI_ASSERT(!read_complete_);
+  header_available_ = true;
+  KALDI_ASSERT(NgramCounts().size() <= kMaxOrder);
+}
+
+void TestableArpaFileParser::ConsumeNGram(const NGram& ngram) {
+  KALDI_ASSERT(header_available_);
+  KALDI_ASSERT(!read_complete_);
+  KALDI_ASSERT(ngram.words.size() <= NgramCounts().size());
+  KALDI_ASSERT(ngram.words.size() >= last_order_);
+  last_order_ = ngram.words.size();
+
+  NGramTestData entry = { 0 };
+  entry.line_number = LineNumber();
+  entry.logprob = ngram.logprob;
+  entry.backoff = ngram.backoff;
+  std::copy(ngram.words.begin(), ngram.words.end(), entry.words);
+  ngrams_.push_back(entry);
+}
+
+void TestableArpaFileParser::ReadComplete() {
+  KALDI_ASSERT(header_available_);
+  KALDI_ASSERT(!read_complete_);
+  read_complete_ = true;
+}
+
+bool CompareNgrams(const NGramTestData &actual,
+                   NGramTestData expected) {
+  expected.logprob *= Log(10.0);
+  expected.backoff *= Log(10.0);
+  if (actual.line_number != expected.line_number
+      || !std::equal(actual.words, actual.words + kMaxOrder,
+                     expected.words)
+      || !ApproxEqual(actual.logprob, expected.logprob)
+      || !ApproxEqual(actual.backoff, expected.backoff)) {
+    KALDI_WARN << "Actual n-gram [" << actual
+               << "] differs from expected [" << expected << "]";
+    return false;
+  }
+  return true;
+}
+
+void TestableArpaFileParser::Validate(
+    CountedArray<int32> expect_counts,
+    CountedArray<NGramTestData> expect_ngrams) {
+  // This needs better disagnostics probably.
+  KALDI_ASSERT(NgramCounts().size() == expect_counts.count);
+  KALDI_ASSERT(std::equal(NgramCounts().begin(), NgramCounts().end(),
+                          expect_counts.array));
+
+  KALDI_ASSERT(ngrams_.size() == expect_ngrams.count);
+  // auto mpos = std::mismatch(ngrams_.begin(), ngrams_.end(),
+  //                           expect_ngrams.array, CompareNgrams);
+  // if (mpos.first != ngrams_.end())
+  //   KALDI_ERR << "Maismatch at index " << mpos.first - ngrams_.begin();
+  // TODO: auto above requres C++11, and I cannot spell out the type!!!
+  KALDI_ASSERT(std::equal(ngrams_.begin(), ngrams_.end(),
+                          expect_ngrams.array, CompareNgrams));
+}
+
+// Read integer LM (no symbols) with log base conversion.
+void ReadIntegerLmLogconvExpectSuccess() {
+  KALDI_LOG << "ReadIntegerLmLogconvExpectSuccess()";
+
+  static std::string integer_lm = "\
+\\data\\\n\
+ngram 1=4\n\
+ngram 2=2\n\
+ngram 3=2\n\
+\n\
+\\1-grams:\n\
+-5.2\t4\t-3.3\n\
+-3.4\t5\n\
+0\t1\t-2.5\n\
+-4.3\t2\n\
+\n\
+\\2-grams:\n\
+-1.4\t4 5\t-3.2\n\
+-1.3\t1 4\t-4.2\n\
+\n\
+\\3-grams:\n\
+-0.3\t1 4 5\n\
+-0.2\t4 5 2\n\
+\n\
+\\end\\";
+
+  int32 expect_counts[] = { 4, 2, 2 };
+  NGramTestData expect_ngrams[] = {
+    {  7, -5.2, { 4, 0, 0 }, -3.3 },
+    {  8, -3.4, { 5, 0, 0 },  0.0 },
+    {  9,  0.0, { 1, 0, 0 }, -2.5 },
+    { 10, -4.3, { 2, 0, 0 },  0.0 },
+
+    { 13, -1.4, { 4, 5, 0 }, -3.2 },
+    { 14, -1.3, { 1, 4, 0 }, -4.2 },
+
+    { 17, -0.3, { 1, 4, 5 },  0.0 },
+    { 18, -0.2, { 4, 5, 2 },  0.0 } };
+
+  ArpaParseOptions options;
+  options.bos_symbol = 1;
+  options.eos_symbol = 2;
+
+  TestableArpaFileParser parser(options, NULL);
+  std::istringstream stm(integer_lm, std::ios_base::in);
+  parser.Read(stm);
+  parser.Validate(MakeCountedArray(expect_counts),
+                  MakeCountedArray(expect_ngrams));
+}
+
+// \xCE\xB2 = UTF-8 for Greek beta, to churn some UTF-8 cranks.
+static std::string symbolic_lm = "\
+We also allow random text coming before the \\data\\\n\
+section marker. Even this is ok:\n\
+\n\
+\\1-grams:\n\
+\n\
+and should be ignored before the \\data\\ marker\n\
+is seen alone by itself on a line.\n\
+\n\
+\\data\\\n\
+ngram 1=4\n\
+ngram 2=2\n\
+ngram 3=2\n\
+\n\
+\\1-grams: \n\
+-5.2\ta\t-3.3\n\
+-3.4\t\xCE\xB2\n\
+0.0\t<s>\t-2.5\n\
+-4.3\t</s>\n\
+\n\
+\\2-grams:\t\n\
+-1.5\ta \xCE\xB2\t-3.2\n\
+-1.3\t<s> a\t-4.2\n\
+\n\
+\\3-grams:\n\
+-0.3\t<s> a \xCE\xB2\n\
+-0.2\t<s> a </s>\n\
+\\end\\";
+
+// Symbol table that is created with predefined test symbols, "a" but no "b".
+class TestSymbolTable : public fst::SymbolTable {
+ public:
+  TestSymbolTable() {
+    AddSymbol("<eps>", 0);
+    AddSymbol("<s>", 1);
+    AddSymbol("</s>", 2);
+    AddSymbol("<unk>", 3);
+    AddSymbol("a", 4);
+  }
+};
+
+// Full expected result shared between ReadSymbolicLmNoOovImpl and
+// ReadSymbolicLmWithOovAddToSymbols().
+NGramTestData expect_symbolic_full[] = {
+  { 15, -5.2, { 4, 0, 0 }, -3.3 },
+  { 16, -3.4, { 5, 0, 0 },  0.0 },
+  { 17,  0.0, { 1, 0, 0 }, -2.5 },
+  { 18, -4.3, { 2, 0, 0 },  0.0 },
+
+  { 21, -1.5, { 4, 5, 0 }, -3.2 },
+  { 22, -1.3, { 1, 4, 0 }, -4.2 },
+
+  { 25, -0.3, { 1, 4, 5 },  0.0 },
+  { 26, -0.2, { 1, 4, 2 },  0.0 } };
+
+// This is run with all possible oov setting and yields same result.
+void ReadSymbolicLmNoOovImpl(ArpaParseOptions::OovHandling oov) {
+  int32 expect_counts[] = { 4, 2, 2 };
+  TestSymbolTable symbols;
+  symbols.AddSymbol("\xCE\xB2", 5);
+
+  ArpaParseOptions options;
+  options.bos_symbol = 1;
+  options.eos_symbol = 2;
+  options.unk_symbol = 3;
+  options.oov_handling = oov;
+  TestableArpaFileParser parser(options, &symbols);
+  std::istringstream stm(symbolic_lm, std::ios_base::in);
+  parser.Read(stm);
+  parser.Validate(MakeCountedArray(expect_counts),
+                  MakeCountedArray(expect_symbolic_full));
+  KALDI_ASSERT(symbols.NumSymbols() == 6);
+}
+
+void ReadSymbolicLmNoOovTests() {
+  KALDI_LOG << "ReadSymbolicLmNoOovImpl(kRaiseError)";
+  ReadSymbolicLmNoOovImpl(ArpaParseOptions::kRaiseError);
+  KALDI_LOG << "ReadSymbolicLmNoOovImpl(kAddToSymbols)";
+  ReadSymbolicLmNoOovImpl(ArpaParseOptions::kAddToSymbols);
+  KALDI_LOG << "ReadSymbolicLmNoOovImpl(kReplaceWithUnk)";
+  ReadSymbolicLmNoOovImpl(ArpaParseOptions::kReplaceWithUnk);
+  KALDI_LOG << "ReadSymbolicLmNoOovImpl(kSkipNGram)";
+  ReadSymbolicLmNoOovImpl(ArpaParseOptions::kSkipNGram);
+}
+
+// This is run with all possible oov setting and yields same result.
+void ReadSymbolicLmWithOovImpl(
+    ArpaParseOptions::OovHandling oov,
+    CountedArray<NGramTestData> expect_ngrams,
+    fst::SymbolTable* symbols) {
+  int32 expect_counts[] = { 4, 2, 2 };
+  ArpaParseOptions options;
+  options.bos_symbol = 1;
+  options.eos_symbol = 2;
+  options.unk_symbol = 3;
+  options.oov_handling = oov;
+  TestableArpaFileParser parser(options, symbols);
+  std::istringstream stm(symbolic_lm, std::ios_base::in);
+  parser.Read(stm);
+  parser.Validate(MakeCountedArray(expect_counts), expect_ngrams);
+}
+
+void ReadSymbolicLmWithOovAddToSymbols() {
+  TestSymbolTable symbols;
+  ReadSymbolicLmWithOovImpl(ArpaParseOptions::kAddToSymbols,
+                            MakeCountedArray(expect_symbolic_full),
+                            &symbols);
+  KALDI_ASSERT(symbols.NumSymbols() == 6);
+  KALDI_ASSERT(symbols.Find("\xCE\xB2") == 5);
+}
+
+void ReadSymbolicLmWithOovReplaceWithUnk() {
+  NGramTestData expect_symbolic_unk_b[] = {
+    { 15, -5.2, { 4, 0, 0 }, -3.3 },
+    { 16, -3.4, { 3, 0, 0 },  0.0 },
+    { 17,  0.0, { 1, 0, 0 }, -2.5 },
+    { 18, -4.3, { 2, 0, 0 },  0.0 },
+
+    { 21, -1.5, { 4, 3, 0 }, -3.2 },
+    { 22, -1.3, { 1, 4, 0 }, -4.2 },
+
+    { 25, -0.3, { 1, 4, 3 },  0.0 },
+    { 26, -0.2, { 1, 4, 2 },  0.0 } };
+
+  TestSymbolTable symbols;
+  ReadSymbolicLmWithOovImpl(ArpaParseOptions::kReplaceWithUnk,
+                            MakeCountedArray(expect_symbolic_unk_b),
+                            &symbols);
+  KALDI_ASSERT(symbols.NumSymbols() == 5);
+}
+
+void ReadSymbolicLmWithOovSkipNGram() {
+  NGramTestData expect_symbolic_no_b[] = {
+    { 15, -5.2, { 4, 0, 0 }, -3.3 },
+    { 17,  0.0, { 1, 0, 0 }, -2.5 },
+    { 18, -4.3, { 2, 0, 0 },  0.0 },
+
+    { 22, -1.3, { 1, 4, 0 }, -4.2 },
+
+    { 26, -0.2, { 1, 4, 2 },  0.0 } };
+
+  TestSymbolTable symbols;
+  ReadSymbolicLmWithOovImpl(ArpaParseOptions::kSkipNGram,
+                            MakeCountedArray(expect_symbolic_no_b),
+                            &symbols);
+  KALDI_ASSERT(symbols.NumSymbols() == 5);
+}
+
+void ReadSymbolicLmWithOovTests() {
+  KALDI_LOG << "ReadSymbolicLmWithOovAddToSymbols()";
+  ReadSymbolicLmWithOovAddToSymbols();
+  KALDI_LOG << "ReadSymbolicLmWithOovReplaceWithUnk()";
+  ReadSymbolicLmWithOovReplaceWithUnk();
+  KALDI_LOG << "ReadSymbolicLmWithOovSkipNGram()";
+  ReadSymbolicLmWithOovSkipNGram();
+}
+
+}  // namespace
+}  // namespace kaldi
+
+int main(int argc, char *argv[]) {
+  kaldi::ReadIntegerLmLogconvExpectSuccess();
+  kaldi::ReadSymbolicLmNoOovTests();
+  kaldi::ReadSymbolicLmWithOovTests();
+}

runtime/onnxruntime/third_party/kaldi/lm/arpa-file-parser.cc

@@ -0,0 +1,281 @@
+// lm/arpa-file-parser.cc
+
+// Copyright 2014  Guoguo Chen
+// Copyright 2016  Smart Action Company LLC (kkm)
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fst/fstlib.h>
+
+#include <sstream>
+
+#include "base/kaldi-error.h"
+#include "base/kaldi-math.h"
+#include "lm/arpa-file-parser.h"
+#include "util/text-utils.h"
+
+namespace kaldi {
+
+ArpaFileParser::ArpaFileParser(const ArpaParseOptions& options,
+                               fst::SymbolTable* symbols)
+    : options_(options), symbols_(symbols),
+      line_number_(0), warning_count_(0) {
+}
+
+ArpaFileParser::~ArpaFileParser() {
+}
+
+void TrimTrailingWhitespace(std::string *str) {
+  str->erase(str->find_last_not_of(" \n\r\t") + 1);
+}
+
+void ArpaFileParser::Read(std::istream &is) {
+  // Argument sanity checks.
+  if (options_.bos_symbol <= 0 || options_.eos_symbol <= 0 ||
+      options_.bos_symbol == options_.eos_symbol)
+    KALDI_ERR << "BOS and EOS symbols are required, must not be epsilons, and "
+              << "differ from each other. Given:"
+              << " BOS=" << options_.bos_symbol
+              << " EOS=" << options_.eos_symbol;
+  if (symbols_ != NULL &&
+      options_.oov_handling == ArpaParseOptions::kReplaceWithUnk &&
+      (options_.unk_symbol <= 0 ||
+       options_.unk_symbol == options_.bos_symbol ||
+       options_.unk_symbol == options_.eos_symbol))
+    KALDI_ERR << "When symbol table is given and OOV mode is kReplaceWithUnk, "
+              << "UNK symbol is required, must not be epsilon, and "
+              << "differ from both BOS and EOS symbols. Given:"
+              << " UNK=" << options_.unk_symbol
+              << " BOS=" << options_.bos_symbol
+              << " EOS=" << options_.eos_symbol;
+  if (symbols_ != NULL && symbols_->Find(options_.bos_symbol).empty())
+    KALDI_ERR << "BOS symbol must exist in symbol table";
+  if (symbols_ != NULL && symbols_->Find(options_.eos_symbol).empty())
+    KALDI_ERR << "EOS symbol must exist in symbol table";
+  if (symbols_ != NULL && options_.unk_symbol > 0 &&
+      symbols_->Find(options_.unk_symbol).empty())
+    KALDI_ERR << "UNK symbol must exist in symbol table";
+
+  ngram_counts_.clear();
+  line_number_ = 0;
+  warning_count_ = 0;
+  current_line_.clear();
+
+#define PARSE_ERR KALDI_ERR << LineReference() << ": "
+
+  // Give derived class an opportunity to prepare its state.
+  ReadStarted();
+
+  // Processes "\data\" section.
+  bool keyword_found = false;
+  while (++line_number_, getline(is, current_line_) && !is.eof()) {
+    if (current_line_.find_first_not_of(" \t\n\r") == std::string::npos) {
+      continue;
+    }
+
+    TrimTrailingWhitespace(&current_line_);
+
+    // Continue skipping lines until the \data\ marker alone on a line is found.
+    if (!keyword_found) {
+      if (current_line_ == "\\data\\") {
+        KALDI_LOG << "Reading \\data\\ section.";
+        keyword_found = true;
+      }
+      continue;
+    }
+
+    if (current_line_[0] == '\\') break;
+
+    // Enters "\data\" section, and looks for patterns like "ngram 1=1000",
+    // which means there are 1000 unigrams.
+    std::size_t equal_symbol_pos = current_line_.find("=");
+    if (equal_symbol_pos != std::string::npos)
+      // Guaranteed spaces around the "=".
+      current_line_.replace(equal_symbol_pos, 1, " = ");
+    std::vector<std::string> col;
+    SplitStringToVector(current_line_, " \t", true, &col);
+    if (col.size() == 4 && col[0] == "ngram" && col[2] == "=") {
+      int32 order, ngram_count = 0;
+      if (!ConvertStringToInteger(col[1], &order) ||
+          !ConvertStringToInteger(col[3], &ngram_count)) {
+        PARSE_ERR << "cannot parse ngram count";
+      }
+      if (ngram_counts_.size() <= order) {
+        ngram_counts_.resize(order);
+      }
+      ngram_counts_[order - 1] = ngram_count;
+    } else {
+      KALDI_WARN << LineReference()
+                 << ": uninterpretable line in \\data\\ section";
+    }
+  }
+
+  if (ngram_counts_.size() == 0)
+    PARSE_ERR << "\\data\\ section missing or empty.";
+
+  // Signal that grammar order and n-gram counts are known.
+  HeaderAvailable();
+
+  NGram ngram;
+  ngram.words.reserve(ngram_counts_.size());
+
+  // Processes "\N-grams:" section.
+  for (int32 cur_order = 1; cur_order <= ngram_counts_.size(); ++cur_order) {
+    // Skips n-grams with zero count.
+    if (ngram_counts_[cur_order - 1] == 0)
+      KALDI_WARN << "Zero ngram count in ngram order " << cur_order
+                 << "(look for 'ngram " << cur_order << "=0' in the \\data\\ "
+                 << " section). There is possibly a problem with the file.";
+
+    // Must be looking at a \k-grams: directive at this point.
+    std::ostringstream keyword;
+    keyword << "\\" << cur_order << "-grams:";
+    if (current_line_ != keyword.str()) {
+      PARSE_ERR << "invalid directive, expecting '" << keyword.str() << "'";
+    }
+    KALDI_LOG << "Reading " << current_line_ << " section.";
+
+    int32 ngram_count = 0;
+    while (++line_number_, getline(is, current_line_) && !is.eof()) {
+      if (current_line_.find_first_not_of(" \n\t\r") == std::string::npos) {
+        continue;
+      }
+      if (current_line_[0] == '\\') {
+        TrimTrailingWhitespace(&current_line_);
+        std::ostringstream next_keyword;
+        next_keyword << "\\" << cur_order + 1 << "-grams:";
+        if ((current_line_ != next_keyword.str()) &&
+            (current_line_ != "\\end\\")) {
+          if (ShouldWarn()) {
+            KALDI_WARN << "ignoring possible directive '" << current_line_
+                       << "' expecting '" << next_keyword.str() << "'";
+
+            if (warning_count_ > 0 &&
+                warning_count_ > static_cast<uint32>(options_.max_warnings)) {
+              KALDI_WARN << "Of " << warning_count_ << " parse warnings, "
+                         << options_.max_warnings << " were reported. "
+                         << "Run program with --max-arpa-warnings=-1 "
+                         << "to see all warnings";
+            }
+          }
+        } else {
+          break;
+        }
+      }
+
+      std::vector<std::string> col;
+      SplitStringToVector(current_line_, " \t", true, &col);
+
+      if (col.size() < 1 + cur_order ||
+          col.size() > 2 + cur_order ||
+          (cur_order == ngram_counts_.size() && col.size() != 1 + cur_order)) {
+        PARSE_ERR << "Invalid n-gram data line";
+      }
+      ++ngram_count;
+
+      // Parse out n-gram logprob and, if present, backoff weight.
+      if (!ConvertStringToReal(col[0], &ngram.logprob)) {
+        PARSE_ERR << "invalid n-gram logprob '" << col[0] << "'";
+      }
+      ngram.backoff = 0.0;
+      if (col.size() > cur_order + 1) {
+        if (!ConvertStringToReal(col[cur_order + 1], &ngram.backoff))
+          PARSE_ERR << "invalid backoff weight '" << col[cur_order + 1] << "'";
+      }
+      // Convert to natural log.
+      ngram.logprob *= M_LN10;
+      ngram.backoff *= M_LN10;
+
+      ngram.words.resize(cur_order);
+      bool skip_ngram = false;
+      for (int32 index = 0; !skip_ngram && index < cur_order; ++index) {
+        int32 word;
+        if (symbols_) {
+          // Symbol table provided, so symbol labels are expected.
+          if (options_.oov_handling == ArpaParseOptions::kAddToSymbols) {
+            word = symbols_->AddSymbol(col[1 + index]);
+          } else {
+            word = symbols_->Find(col[1 + index]);
+            if (word == -1) { // fst::kNoSymbol
+              switch (options_.oov_handling) {
+                case ArpaParseOptions::kReplaceWithUnk:
+                  word = options_.unk_symbol;
+                  break;
+                case ArpaParseOptions::kSkipNGram:
+                  if (ShouldWarn())
+                    KALDI_WARN << LineReference() << " skipped: word '"
+                               << col[1 + index] << "' not in symbol table";
+                  skip_ngram = true;
+                  break;
+                default:
+                  PARSE_ERR << "word '"  << col[1 + index]
+                            << "' not in symbol table";
+              }
+            }
+          }
+        } else {
+          // Symbols not provided, LM file should contain integers.
+          if (!ConvertStringToInteger(col[1 + index], &word) || word < 0) {
+            PARSE_ERR << "invalid symbol '" << col[1 + index] << "'";
+          }
+        }
+        // Whichever way we got it, an epsilon is invalid.
+        if (word == 0) {
+          PARSE_ERR << "epsilon symbol '" << col[1 + index]
+                    << "' is illegal in ARPA LM";
+        }
+        ngram.words[index] = word;
+      }
+      if (!skip_ngram) {
+        ConsumeNGram(ngram);
+      }
+    }
+    if (ngram_count > ngram_counts_[cur_order - 1]) {
+      PARSE_ERR << "header said there would be " << ngram_counts_[cur_order - 1]
+                << " n-grams of order " << cur_order
+                << ", but we saw more already.";
+    }
+  }
+
+  if (current_line_ != "\\end\\") {
+    PARSE_ERR << "invalid or unexpected directive line, expecting \\end\\";
+  }
+
+  if (warning_count_ > 0 &&
+      warning_count_ > static_cast<uint32>(options_.max_warnings)) {
+    KALDI_WARN << "Of " << warning_count_ << " parse warnings, "
+               << options_.max_warnings << " were reported. Run program with "
+               << "--max-arpa-warnings=-1 to see all warnings";
+  }
+
+  current_line_.clear();
+  ReadComplete();
+
+#undef PARSE_ERR
+}
+
+std::string ArpaFileParser::LineReference() const {
+  std::ostringstream ss;
+  ss << "line " << line_number_ << " [" << current_line_ << "]";
+  return ss.str();
+}
+
+bool ArpaFileParser::ShouldWarn() {
+  return (warning_count_ != -1) &&
+    (++warning_count_ <= static_cast<uint32>(options_.max_warnings));
+}
+
+}  // namespace kaldi

runtime/onnxruntime/third_party/kaldi/lm/arpa-file-parser.h

@@ -0,0 +1,146 @@
+// lm/arpa-file-parser.h
+
+// Copyright 2014  Guoguo Chen
+// Copyright 2016  Smart Action Company LLC (kkm)
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef KALDI_LM_ARPA_FILE_PARSER_H_
+#define KALDI_LM_ARPA_FILE_PARSER_H_
+
+#include <fst/fst-decl.h>
+
+#include <string>
+#include <vector>
+
+#include "base/kaldi-types.h"
+#include "itf/options-itf.h"
+
+namespace kaldi {
+
+/**
+  Options that control ArpaFileParser
+*/
+struct ArpaParseOptions {
+  enum OovHandling {
+    kRaiseError,     ///< Abort on OOV words
+    kAddToSymbols,   ///< Add novel words to the symbol table.
+    kReplaceWithUnk,  ///< Replace OOV words with <unk>.
+    kSkipNGram       ///< Skip n-gram with OOV word and continue.
+  };
+
+  ArpaParseOptions():
+      bos_symbol(-1), eos_symbol(-1), unk_symbol(-1),
+      oov_handling(kRaiseError), max_warnings(30) { }
+
+  void Register(OptionsItf *opts) {
+    // Registering only the max_warnings count, since other options are
+    // treated differently by client programs: some want integer symbols,
+    // while other are passed words in their command line.
+    opts->Register("max-arpa-warnings", &max_warnings,
+                   "Maximum warnings to report on ARPA parsing, "
+                   "0 to disable, -1 to show all");
+  }
+
+  int32 bos_symbol;  ///< Symbol for <s>, Required non-epsilon.
+  int32 eos_symbol;  ///< Symbol for </s>, Required non-epsilon.
+  int32 unk_symbol;  ///< Symbol for <unk>, Required for kReplaceWithUnk.
+  OovHandling oov_handling;  ///< How to handle OOV words in the file.
+  int32 max_warnings;  ///< Maximum warnings to report, <0 unlimited.
+};
+
+/**
+   A parsed n-gram from ARPA LM file.
+*/
+struct NGram {
+  NGram() : logprob(0.0), backoff(0.0) { }
+  std::vector<int32> words;  ///< Symbols in left to right order.
+  float logprob;             ///< Log-prob of the n-gram.
+  float backoff;             ///< log-backoff weight of the n-gram.
+                             ///< Defaults to zero if not specified.
+};
+
+/**
+    ArpaFileParser is an abstract base class for ARPA LM file conversion.
+
+    See ConstArpaLmBuilder and ArpaLmCompiler for usage examples.
+*/
+class ArpaFileParser {
+ public:
+  /// Constructs the parser with the given options and optional symbol table.
+  /// If symbol table is provided, then the file should contain text n-grams,
+  /// and the words are mapped to symbols through it. bos_symbol and
+  /// eos_symbol in the options structure must be valid symbols in the table,
+  /// and so must be unk_symbol if provided. The table is not owned by the
+  /// parser, but may be augmented, if oov_handling is set to kAddToSymbols.
+  /// If symbol table is a null pointer, the file should contain integer
+  /// symbol values, and oov_handling has no effect. bos_symbol and eos_symbol
+  /// must be valid symbols still.
+  ArpaFileParser(const ArpaParseOptions& options, fst::SymbolTable* symbols);
+  virtual ~ArpaFileParser();
+
+  /// Read ARPA LM file from a stream.
+  void Read(std::istream &is);
+
+  /// Parser options.
+  const ArpaParseOptions& Options() const { return options_; }
+
+ protected:
+  /// Override called before reading starts. This is the point to prepare
+  /// any state in the derived class.
+  virtual void ReadStarted() { }
+
+  /// Override function called to signal that ARPA header with the expected
+  /// number of n-grams has been read, and ngram_counts() is now valid.
+  virtual void HeaderAvailable() { }
+
+  /// Pure override that must be implemented to process current n-gram. The
+  /// n-grams are sent in the file order, which guarantees that all
+  /// (k-1)-grams are processed before the first k-gram is.
+  virtual void ConsumeNGram(const NGram&) = 0;
+
+  /// Override function called after the last n-gram has been consumed.
+  virtual void ReadComplete() { }
+
+  /// Read-only access to symbol table. Not owned, do not make public.
+  const fst::SymbolTable* Symbols() const { return symbols_; }
+
+  /// Inside ConsumeNGram(), provides the current line number.
+  int32 LineNumber() const { return line_number_; }
+
+  /// Inside ConsumeNGram(), returns a formatted reference to the line being
+  /// compiled, to print out as part of diagnostics.
+  std::string LineReference() const;
+
+  /// Increments warning count, and returns true if a warning should be
+  /// printed or false if the count has exceeded the set maximum.
+  bool ShouldWarn();
+
+  /// N-gram counts. Valid from the point when HeaderAvailable() is called.
+  const std::vector<int32>& NgramCounts() const { return ngram_counts_; }
+
+ private:
+  ArpaParseOptions options_;
+  fst::SymbolTable* symbols_;  // the pointer is not owned here.
+  int32 line_number_;
+  uint32 warning_count_;
+  std::string current_line_;
+  std::vector<int32> ngram_counts_;
+};
+
+}  // namespace kaldi
+
+#endif  // KALDI_LM_ARPA_FILE_PARSER_H_

runtime/onnxruntime/third_party/kaldi/lm/arpa-lm-compiler-test.cc

@@ -0,0 +1,250 @@
+// lm/arpa-lm-compiler-test.cc
+
+// Copyright 2009-2011 Gilles Boulianne
+// Copyright 2016 Smart Action LLC (kkm)
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iostream>
+#include <string>
+#include <sstream>
+
+#include "base/kaldi-error.h"
+#include "base/kaldi-math.h"
+#include "lm/arpa-lm-compiler.h"
+#include "util/kaldi-io.h"
+
+namespace kaldi {
+
+// Predefine some symbol values, because any integer is as good than any other.
+enum {
+  kEps = 0,
+  kDisambig,
+  kBos, kEos,
+};
+
+// Number of random sentences for coverage test.
+static const int kRandomSentences = 50;
+
+// Creates an FST that generates any sequence of symbols taken from given
+// symbol table. The FST is then associated with the symbol table.
+static fst::StdVectorFst* CreateGenFst(bool seps, const fst::SymbolTable* pst) {
+  fst::StdVectorFst* genFst = new fst::StdVectorFst;
+  genFst->SetInputSymbols(pst);
+  genFst->SetOutputSymbols(pst);
+
+  fst::StdArc::StateId midId   = genFst->AddState();
+  if (!seps) {
+    fst::StdArc::StateId initId  = genFst->AddState();
+    fst::StdArc::StateId finalId = genFst->AddState();
+    genFst->SetStart(initId);
+    genFst->SetFinal(finalId, fst::StdArc::Weight::One());
+    genFst->AddArc(initId, fst::StdArc(kBos, kBos, 0, midId));
+    genFst->AddArc(midId,  fst::StdArc(kEos, kEos, 0, finalId));
+  } else {
+    genFst->SetStart(midId);
+    genFst->SetFinal(midId, fst::StdArc::Weight::One());
+  }
+
+  // Add a loop for each symbol in the table except the four special ones.
+  fst::SymbolTableIterator si(*pst);
+  for (si.Reset(); !si.Done(); si.Next()) {
+    if (si.Value() == kBos || si.Value() == kEos ||
+        si.Value() == kEps || si.Value() == kDisambig)
+      continue;
+    genFst->AddArc(midId, fst::StdArc(si.Value(), si.Value(),
+                                      fst::StdArc::Weight::One(), midId));
+  }
+  return genFst;
+}
+
+// Compile given ARPA file.
+ArpaLmCompiler* Compile(bool seps, const std::string &infile) {
+  ArpaParseOptions options;
+  fst::SymbolTable symbols;
+  // Use spaces on special symbols, so we rather fail than read them by mistake.
+  symbols.AddSymbol(" <eps>", kEps);
+  symbols.AddSymbol(" #0", kDisambig);
+  options.bos_symbol = symbols.AddSymbol("<s>", kBos);
+  options.eos_symbol = symbols.AddSymbol("</s>", kEos);
+  options.oov_handling = ArpaParseOptions::kAddToSymbols;
+
+  // Tests in this form cannot be run with epsilon substitution, unless every
+  // random path is also fitted with a #0-transducing self-loop.
+  ArpaLmCompiler* lm_compiler =
+      new ArpaLmCompiler(options,
+                         seps ? kDisambig : 0,
+                         &symbols);
+  {
+    Input ki(infile);
+    lm_compiler->Read(ki.Stream());
+  }
+  return lm_compiler;
+}
+
+// Add a state to an FSA after last_state, add a form last_state to the new
+// state, and return the new state.
+fst::StdArc::StateId AddToChainFsa(fst::StdMutableFst* fst,
+                                   fst::StdArc::StateId last_state,
+                                   int64 symbol) {
+  fst::StdArc::StateId next_state  = fst->AddState();
+  fst->AddArc(last_state, fst::StdArc(symbol, symbol, 0, next_state));
+  return next_state;
+}
+
+// Add a disambiguator-generating self loop to every state of an FST.
+void AddSelfLoops(fst::StdMutableFst* fst) {
+  for (fst::StateIterator<fst::StdMutableFst> siter(*fst);
+       !siter.Done(); siter.Next()) {
+    fst->AddArc(siter.Value(),
+                fst::StdArc(kEps, kDisambig, 0, siter.Value()));
+  }
+}
+
+// Compiles infile and then runs kRandomSentences random coverage tests on the
+// compiled FST.
+bool CoverageTest(bool seps, const std::string &infile) {
+  // Compile ARPA model.
+  ArpaLmCompiler* lm_compiler = Compile(seps, infile);
+
+  // Create an FST that generates any sequence of symbols taken from the model
+  // output.
+  fst::StdVectorFst* genFst =
+      CreateGenFst(seps, lm_compiler->Fst().OutputSymbols());
+
+  int num_successes = 0;
+  for (int32 i = 0; i < kRandomSentences; ++i) {
+    // Generate a random sentence FST.
+    fst::StdVectorFst sentence;
+    RandGen(*genFst, &sentence);
+    if (seps)
+      AddSelfLoops(&sentence);
+
+    fst::ArcSort(lm_compiler->MutableFst(), fst::StdOLabelCompare());
+
+    // The past must successfully compose with the LM FST.
+    fst::StdVectorFst composition;
+    Compose(sentence, lm_compiler->Fst(), &composition);
+    if (composition.Start() != fst::kNoStateId)
+      ++num_successes;
+  }
+
+  delete genFst;
+  delete lm_compiler;
+
+  bool ok = num_successes == kRandomSentences;
+  if (!ok) {
+    KALDI_WARN << "Coverage test failed on " << infile << ": composed "
+               << num_successes << "/" << kRandomSentences;
+  }
+  return ok;
+}
+
+bool ScoringTest(bool seps, const std::string &infile, const std::string& sentence,
+                 float expected) {
+  ArpaLmCompiler* lm_compiler = Compile(seps, infile);
+  const fst::SymbolTable* symbols = lm_compiler->Fst().InputSymbols();
+
+  // Create a sentence FST for scoring.
+  fst::StdVectorFst sentFst;
+  fst::StdArc::StateId state = sentFst.AddState();
+  sentFst.SetStart(state);
+  if (!seps) {
+    state = AddToChainFsa(&sentFst, state, kBos);
+  }
+  std::stringstream ss(sentence);
+  std::string word;
+  while (ss >> word) {
+    int64 word_sym = symbols->Find(word);
+    KALDI_ASSERT(word_sym != -1);
+    state = AddToChainFsa(&sentFst, state, word_sym);
+  }
+  if (!seps) {
+    state = AddToChainFsa(&sentFst, state, kEos);
+  }
+  if (seps) {
+    AddSelfLoops(&sentFst);
+  }
+  sentFst.SetFinal(state, 0);
+  sentFst.SetOutputSymbols(symbols);
+
+  // Do the composition and extract final weight.
+  fst::StdVectorFst composed;
+  fst::Compose(sentFst, lm_compiler->Fst(), &composed);
+  delete lm_compiler;
+
+  if (composed.Start() == fst::kNoStateId) {
+    KALDI_WARN << "Test sentence " << sentence << " did not compose "
+               << "with the language model FST\n";
+    return false;
+  }
+
+  std::vector<fst::StdArc::Weight> shortest;
+  fst::ShortestDistance(composed, &shortest, true);
+  float actual = shortest[composed.Start()].Value();
+
+  bool ok = ApproxEqual(expected, actual);
+  if (!ok) {
+    KALDI_WARN << "Scored " << sentence << " in " << infile
+               << ": Expected=" << expected << " actual=" << actual;
+  }
+  return ok;
+}
+
+bool ThrowsExceptionTest(bool seps, const std::string &infile) {
+  try {
+    // Make memory cleanup easy in both cases of try-catch block.
+    std::unique_ptr<ArpaLmCompiler> compiler(Compile(seps, infile));
+    return false;
+  } catch (const KaldiFatalError&) {
+    return true;
+  }
+}
+
+}  // namespace kaldi
+
+bool RunAllTests(bool seps) {
+  bool ok = true;
+  ok &= kaldi::CoverageTest(seps, "test_data/missing_backoffs.arpa");
+  ok &= kaldi::CoverageTest(seps, "test_data/unused_backoffs.arpa");
+  ok &= kaldi::CoverageTest(seps, "test_data/input.arpa");
+
+  ok &= kaldi::ScoringTest(seps, "test_data/input.arpa", "b b b a", 59.2649);
+  ok &= kaldi::ScoringTest(seps, "test_data/input.arpa", "a b", 4.36082);
+
+  ok &= kaldi::ThrowsExceptionTest(seps, "test_data/missing_bos.arpa");
+
+  if (!ok) {
+    KALDI_WARN << "Tests " << (seps ? "with" : "without")
+               << " epsilon substitution FAILED";
+  }
+  return ok;
+}
+
+int main(int argc, char *argv[]) {
+  bool ok = true;
+
+  ok &= RunAllTests(false);  // Without disambiguators (old behavior).
+  ok &= RunAllTests(true);   // With epsilon substitution (new behavior).
+
+  if (ok) {
+    KALDI_LOG << "All tests passed";
+    return 0;
+  } else {
+    KALDI_WARN << "Test FAILED";
+    return 1;
+  }
+}

runtime/onnxruntime/third_party/kaldi/lm/arpa-lm-compiler.cc

@@ -0,0 +1,377 @@
+// lm/arpa-lm-compiler.cc
+
+// Copyright 2009-2011 Gilles Boulianne
+// Copyright 2016 Smart Action LLC (kkm)
+// Copyright 2017 Xiaohui Zhang
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <limits>
+#include <sstream>
+#include <utility>
+
+#include "base/kaldi-math.h"
+#include "lm/arpa-lm-compiler.h"
+#include "util/stl-utils.h"
+#include "util/text-utils.h"
+#include "fstext/remove-eps-local.h"
+
+namespace kaldi {
+
+class ArpaLmCompilerImplInterface {
+ public:
+  virtual ~ArpaLmCompilerImplInterface() { }
+  virtual void ConsumeNGram(const NGram& ngram, bool is_highest) = 0;
+};
+
+namespace {
+
+typedef int32 StateId;
+typedef int32 Symbol;
+
+// GeneralHistKey can represent state history in an arbitrarily large n
+// n-gram model with symbol ids fitting int32.
+class GeneralHistKey {
+ public:
+  // Construct key from being and end iterators.
+  template<class InputIt>
+  GeneralHistKey(InputIt begin, InputIt end) : vector_(begin, end) { }
+  // Construct empty history key.
+  GeneralHistKey() : vector_() { }
+  // Return tails of the key as a GeneralHistKey. The tails of an n-gram
+  // w[1..n] is the sequence w[2..n] (and the heads is w[1..n-1], but the
+  // key class does not need this operartion).
+  GeneralHistKey Tails() const {
+    return GeneralHistKey(vector_.begin() + 1, vector_.end());
+  }
+  // Keys are equal if represent same state.
+  friend bool operator==(const GeneralHistKey& a, const GeneralHistKey& b) {
+    return a.vector_ == b.vector_;
+  }
+  // Public typename HashType for hashing.
+  struct HashType {
+    size_t operator()(const GeneralHistKey& key) const {
+      return VectorHasher<Symbol>().operator()(key.vector_);
+    }
+  };
+
+ private:
+  std::vector<Symbol> vector_;
+};
+
+// OptimizedHistKey combines 3 21-bit symbol ID values into one 64-bit
+// machine word. allowing significant memory reduction and some runtime
+// benefit over GeneralHistKey. Since 3 symbols are enough to track history
+// in a 4-gram model, this optimized key is used for smaller models with up
+// to 4-gram and symbol values up to 2^21-1.
+//
+// See GeneralHistKey for interface requirements of a key class.
+class OptimizedHistKey {
+ public:
+  enum {
+    kShift = 21,  // 21 * 3 = 63 bits for data.
+    kMaxData = (1 << kShift) - 1
+  };
+  template<class InputIt>
+  OptimizedHistKey(InputIt begin, InputIt end) : data_(0) {
+    for (uint32 shift = 0; begin != end; ++begin, shift += kShift) {
+      data_ |= static_cast<uint64>(*begin) << shift;
+    }
+  }
+  OptimizedHistKey() : data_(0) { }
+  OptimizedHistKey Tails() const {
+    return OptimizedHistKey(data_ >> kShift);
+  }
+  friend bool operator==(const OptimizedHistKey& a, const OptimizedHistKey& b) {
+    return a.data_ == b.data_;
+  }
+  struct HashType {
+    size_t operator()(const OptimizedHistKey& key) const { return key.data_; }
+  };
+
+ private:
+  explicit OptimizedHistKey(uint64 data) : data_(data) { }
+  uint64 data_;
+};
+
+}  // namespace
+
+template <class HistKey>
+class ArpaLmCompilerImpl : public ArpaLmCompilerImplInterface {
+ public:
+  ArpaLmCompilerImpl(ArpaLmCompiler* parent, fst::StdVectorFst* fst,
+                     Symbol sub_eps);
+
+  virtual void ConsumeNGram(const NGram &ngram, bool is_highest);
+
+ private:
+  StateId AddStateWithBackoff(HistKey key, float backoff);
+  void CreateBackoff(HistKey key, StateId state, float weight);
+
+  ArpaLmCompiler *parent_;  // Not owned.
+  fst::StdVectorFst* fst_;  // Not owned.
+  Symbol bos_symbol_;
+  Symbol eos_symbol_;
+  Symbol sub_eps_;
+
+  StateId eos_state_;
+  typedef unordered_map<HistKey, StateId,
+                        typename HistKey::HashType> HistoryMap;
+  HistoryMap history_;
+};
+
+template <class HistKey>
+ArpaLmCompilerImpl<HistKey>::ArpaLmCompilerImpl(
+    ArpaLmCompiler* parent, fst::StdVectorFst* fst, Symbol sub_eps)
+    : parent_(parent), fst_(fst), bos_symbol_(parent->Options().bos_symbol),
+      eos_symbol_(parent->Options().eos_symbol), sub_eps_(sub_eps) {
+  // The algorithm maintains state per history. The 0-gram is a special state
+  // for empty history. All unigrams (including BOS) backoff into this state.
+  StateId zerogram = fst_->AddState();
+  history_[HistKey()] = zerogram;
+
+  // Also, if </s> is not treated as epsilon, create a common end state for
+  // all transitions accepting the </s>, since they do not back off. This small
+  // optimization saves about 2% states in an average grammar.
+  if (sub_eps_ == 0) {
+    eos_state_ = fst_->AddState();
+    fst_->SetFinal(eos_state_, 0);
+  }
+}
+
+template <class HistKey>
+void ArpaLmCompilerImpl<HistKey>::ConsumeNGram(const NGram &ngram,
+                                               bool is_highest) {
+  // Generally, we do the following. Suppose we are adding an n-gram "A B
+  // C". Then find the node for "A B", add a new node for "A B C", and connect
+  // them with the arc accepting "C" with the specified weight. Also, add a
+  // backoff arc from the new "A B C" node to its backoff state "B C".
+  //
+  // Two notable exceptions are the highest order n-grams, and final n-grams.
+  //
+  // When adding a highest order n-gram (e. g., our "A B C" is in a 3-gram LM),
+  // the following optimization is performed. There is no point adding a node
+  // for "A B C" with a "C" arc from "A B", since there will be no other
+  // arcs ingoing to this node, and an epsilon backoff arc into the backoff
+  // model "B C", with the weight of \bar{1}. To save a node, create an arc
+  // accepting "C" directly from "A B" to "B C". This saves as many nodes
+  // as there are the highest order n-grams, which is typically about half
+  // the size of a large 3-gram model.
+  //
+  // Indeed, this does not apply to n-grams ending in EOS, since they do not
+  // back off. These are special, as they do not have a back-off state, and
+  // the node for "(..anything..) </s>" is always final. These are handled
+  // in one of the two possible ways, If symbols <s> and </s> are being
+  // replaced by epsilons, neither node nor arc is created, and the logprob
+  // of the n-gram is applied to its source node as final weight. If <s> and
+  // </s> are preserved, then a special final node for </s> is allocated and
+  // used as the destination of the "</s>" acceptor arc.
+  HistKey heads(ngram.words.begin(), ngram.words.end() - 1);
+  typename HistoryMap::iterator source_it = history_.find(heads);
+  if (source_it == history_.end()) {
+    // There was no "A B", therefore the probability of "A B C" is zero.
+    // Print a warning and discard current n-gram.
+    if (parent_->ShouldWarn())
+      KALDI_WARN << parent_->LineReference()
+                 << " skipped: no parent (n-1)-gram exists";
+    return;
+  }
+
+  StateId source = source_it->second;
+  StateId dest;
+  Symbol sym = ngram.words.back();
+  float weight = -ngram.logprob;
+  if (sym == sub_eps_ || sym == 0) {
+    KALDI_ERR << " <eps> or disambiguation symbol " << sym << "found in the ARPA file. ";
+  }
+  if (sym == eos_symbol_) {
+    if (sub_eps_ == 0) {
+      // Keep </s> as a real symbol when not substituting.
+      dest = eos_state_;
+    } else {
+      // Treat </s> as if it was epsilon: mark source final, with the weight
+      // of the n-gram.
+      fst_->SetFinal(source, weight);
+      return;
+    }
+  } else {
+    // For the highest order n-gram, this may find an existing state, for
+    // non-highest, will create one (unless there are duplicate n-grams
+    // in the grammar, which cannot be reliably detected if highest order,
+    // so we better do not do that at all).
+    dest = AddStateWithBackoff(
+        HistKey(ngram.words.begin() + (is_highest ? 1 : 0),
+                ngram.words.end()),
+        -ngram.backoff);
+  }
+
+  if (sym == bos_symbol_) {
+    weight = 0;  // Accepting <s> is always free.
+    if (sub_eps_ == 0) {
+      // <s> is as a real symbol, only accepted in the start state.
+      source = fst_->AddState();
+      fst_->SetStart(source);
+    } else {
+      // The new state for <s> unigram history *is* the start state.
+      fst_->SetStart(dest);
+      return;
+    }
+  }
+
+  // Add arc from source to dest, whichever way it was found.
+  fst_->AddArc(source, fst::StdArc(sym, sym, weight, dest));
+  return;
+}
+
+// Find or create a new state for n-gram defined by key, and ensure it has a
+// backoff transition.  The key is either the current n-gram for all but
+// highest orders, or the tails of the n-gram for the highest order. The
+// latter arises from the chain-collapsing optimization described above.
+template <class HistKey>
+StateId ArpaLmCompilerImpl<HistKey>::AddStateWithBackoff(HistKey key,
+                                                         float backoff) {
+  typename HistoryMap::iterator dest_it = history_.find(key);
+  if (dest_it != history_.end()) {
+    // Found an existing state in the history map. Invariant: if the state in
+    // the map, then its backoff arc is in the FST. We are done.
+    return dest_it->second;
+  }
+  // Otherwise create a new state and its backoff arc, and register in the map.
+  StateId dest = fst_->AddState();
+  history_[key] = dest;
+  CreateBackoff(key.Tails(), dest, backoff);
+  return dest;
+}
+
+// Create a backoff arc for a state. Key is a backoff destination that may or
+// may not exist. When the destination is not found, naturally fall back to
+// the lower order model, and all the way down until one is found (since the
+// 0-gram model is always present, the search is guaranteed to terminate).
+template <class HistKey>
+inline void ArpaLmCompilerImpl<HistKey>::CreateBackoff(
+    HistKey key, StateId state, float weight) {
+  typename HistoryMap::iterator dest_it = history_.find(key);
+  while (dest_it == history_.end()) {
+    key = key.Tails();
+    dest_it = history_.find(key);
+  }
+
+  // The arc should transduce either <eos> or #0 to <eps>, depending on the
+  // epsilon substitution mode. This is the only case when input and output
+  // label may differ.
+  fst_->AddArc(state, fst::StdArc(sub_eps_, 0, weight, dest_it->second));
+}
+
+ArpaLmCompiler::~ArpaLmCompiler() {
+  if (impl_ != NULL)
+    delete impl_;
+}
+
+void ArpaLmCompiler::HeaderAvailable() {
+  KALDI_ASSERT(impl_ == NULL);
+  // Use optimized implementation if the grammar is 4-gram or less, and the
+  // maximum attained symbol id will fit into the optimized range.
+  int64 max_symbol = 0;
+  if (Symbols() != NULL)
+    max_symbol = Symbols()->AvailableKey() - 1;
+  // If augmenting the symbol table, assume the worst case when all words in
+  // the model being read are novel.
+  if (Options().oov_handling == ArpaParseOptions::kAddToSymbols)
+    max_symbol += NgramCounts()[0];
+
+  if (NgramCounts().size() <= 4 && max_symbol < OptimizedHistKey::kMaxData) {
+    impl_ = new ArpaLmCompilerImpl<OptimizedHistKey>(this, &fst_, sub_eps_);
+  } else {
+    impl_ = new ArpaLmCompilerImpl<GeneralHistKey>(this, &fst_, sub_eps_);
+    KALDI_LOG << "Reverting to slower state tracking because model is large: "
+              << NgramCounts().size() << "-gram with symbols up to "
+              << max_symbol;
+  }
+}
+
+void ArpaLmCompiler::ConsumeNGram(const NGram &ngram) {
+  // <s> is invalid in tails, </s> in heads of an n-gram.
+  for (int i = 0; i < ngram.words.size(); ++i) {
+    if ((i > 0 && ngram.words[i] == Options().bos_symbol) ||
+        (i + 1 < ngram.words.size()
+         && ngram.words[i] == Options().eos_symbol)) {
+      if (ShouldWarn())
+        KALDI_WARN << LineReference()
+                   << " skipped: n-gram has invalid BOS/EOS placement";
+      return;
+    }
+  }
+
+  bool is_highest = ngram.words.size() == NgramCounts().size();
+  impl_->ConsumeNGram(ngram, is_highest);
+}
+
+void ArpaLmCompiler::RemoveRedundantStates() {
+  fst::StdArc::Label backoff_symbol = sub_eps_;
+  if (backoff_symbol == 0) {
+    // The method of removing redundant states implemented in this function
+    // leads to slow determinization of L o G when people use the older style of
+    // usage of arpa2fst where the --disambig-symbol option was not specified.
+    // The issue seems to be that it creates a non-deterministic FST, while G is
+    // supposed to be deterministic.  By 'return'ing below, we just disable this
+    // method if people were using an older script.  This method isn't really
+    // that consequential anyway, and people will move to the newer-style
+    // scripts (see current utils/format_lm.sh), so this isn't much of a
+    // problem.
+    return;
+  }
+
+  fst::StdArc::StateId num_states = fst_.NumStates();
+
+
+  // replace the #0 symbols on the input of arcs out of redundant states (states
+  // that are not final and have only a backoff arc leaving them), with <eps>.
+  for (fst::StdArc::StateId state = 0; state < num_states; state++) {
+    if (fst_.NumArcs(state) == 1 && fst_.Final(state) == fst::TropicalWeight::Zero()) {
+      fst::MutableArcIterator<fst::StdVectorFst> iter(&fst_, state);
+      fst::StdArc arc = iter.Value();
+      if (arc.ilabel == backoff_symbol) {
+        arc.ilabel = 0;
+        iter.SetValue(arc);
+      }
+    }
+  }
+
+  // we could call fst::RemoveEps, and it would have the same effect in normal
+  // cases, where backoff_symbol != 0 and there are no epsilons in unexpected
+  // places, but RemoveEpsLocal is a bit safer in case something weird is going
+  // on; it guarantees not to blow up the FST.
+  fst::RemoveEpsLocal(&fst_);
+  KALDI_LOG << "Reduced num-states from " << num_states << " to "
+            << fst_.NumStates();
+}
+
+void ArpaLmCompiler::Check() const {
+  if (fst_.Start() == fst::kNoStateId) {
+    KALDI_ERR << "Arpa file did not contain the beginning-of-sentence symbol "
+              << Symbols()->Find(Options().bos_symbol) << ".";
+  }
+}
+
+void ArpaLmCompiler::ReadComplete() {
+  fst_.SetInputSymbols(Symbols());
+  fst_.SetOutputSymbols(Symbols());
+  RemoveRedundantStates();
+  Check();
+}
+
+}  // namespace kaldi

runtime/onnxruntime/third_party/kaldi/lm/arpa-lm-compiler.h

@@ -0,0 +1,65 @@
+// lm/arpa-lm-compiler.h
+
+// Copyright 2009-2011 Gilles Boulianne
+// Copyright 2016 Smart Action LLC (kkm)
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef KALDI_LM_ARPA_LM_COMPILER_H_
+#define KALDI_LM_ARPA_LM_COMPILER_H_
+
+#include <fst/fstlib.h>
+
+#include "lm/arpa-file-parser.h"
+
+namespace kaldi {
+
+class ArpaLmCompilerImplInterface;
+
+class ArpaLmCompiler : public ArpaFileParser {
+ public:
+  ArpaLmCompiler(const ArpaParseOptions& options, int sub_eps,
+                 fst::SymbolTable* symbols)
+      : ArpaFileParser(options, symbols),
+        sub_eps_(sub_eps), impl_(NULL) {
+  }
+  ~ArpaLmCompiler();
+
+  const fst::StdVectorFst& Fst() const { return fst_; }
+  fst::StdVectorFst* MutableFst() { return &fst_; }
+
+ protected:
+  // ArpaFileParser overrides.
+  virtual void HeaderAvailable();
+  virtual void ConsumeNGram(const NGram& ngram);
+  virtual void ReadComplete();
+
+
+ private:
+  // this function removes states that only have a backoff arc coming
+  // out of them.
+  void RemoveRedundantStates();
+  void Check() const;
+
+  int sub_eps_;
+  ArpaLmCompilerImplInterface* impl_;  // Owned.
+  fst::StdVectorFst fst_;
+  template <class HistKey> friend class ArpaLmCompilerImpl;
+};
+
+}  // namespace kaldi
+
+#endif  // KALDI_LM_ARPA_LM_COMPILER_H_

runtime/onnxruntime/third_party/kaldi/lm/const-arpa-lm.h

@@ -0,0 +1,430 @@
+// lm/const-arpa-lm.h
+
+// Copyright 2014  Guoguo Chen
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef KALDI_LM_CONST_ARPA_LM_H_
+#define KALDI_LM_CONST_ARPA_LM_H_
+
+#include <string>
+#include <vector>
+
+#include "base/kaldi-common.h"
+#include "fstext/deterministic-fst.h"
+#include "lm/arpa-file-parser.h"
+#include "util/common-utils.h"
+
+namespace kaldi {
+
+/**
+    The following explains how the const arpa LM works. We will start from a toy
+    example, and gradually get to the existing framework. Related classes are:
+    LmState, ConstArpaLmBuilder and ConstArpaLm.
+
+    First, let's explain how we can compute LM scores from an Arpa file. Suppose
+    we want to get the N-gram prob for "A B C". We can code the lookup something
+    very roughly like this:
+
+    float GetNgramLogprob(hist, word) {  // hist = "A B", word = "C"
+      backoff_logprob = 0.0;
+      if ((state = GetLmState(hist)) != NULL) {
+        // "A B" exists as a prefix in the LM
+        if (state->HasWord(word)) {
+          return state->Logprob(word);
+        } else {
+          // We'll need to backoff to "B C", but include the backoff penalty.
+          backoff_logprob = state->BackoffLogprob();
+        }
+      }
+      return backoff_logprob + GetNgramLogprob(hist_minus_first_word, word);
+    }
+
+    In terms of data-structures, in the most abstract form of it would be
+    something like the following (note, we assume words in the lexicon can be
+    represented as int32, and that these indexes are nonnegative):
+
+    class LmState {  // e.g., LmState for "A B"
+      // This is the actual LM-prob of this sequence, e.g. if this state is
+      // "A B" then it would be the logprob of "A -> B".
+      float logprob_;
+
+      // Backoff probability for LM-state "A B" -> "X" backing off to "B" -> "X"
+      // if "A B X" is not present in the language model.
+      float backoff_logprob_;
+
+      // e.g. "C" -> LmState of "A B C".
+      std::unordered_map<int32, LmState*> children_;
+    };
+
+    The above design is very memory inefficient for two reasons:
+    1. Suppose "A B" has no children, i.e. no C such that "A B C" is an n-gram.
+       In this case the backoff_logprob will be zero and the 'children' vector
+       will be empty. So all we need is the "float logprob_". Let's call "A B" a
+       leaf in this case.
+    2. The map std::unordered_map uses a lot of memory.
+
+    A first iteration of making this efficient is to get rid of the map as
+    follows:
+
+    class LmState {
+      float logprob_;
+      float backoff_logprob_;
+      std::vector<std::pair<int32, int32> > children_;
+    };
+
+    Here, the 'children_' vector contains pairs (child_word, child_info), sorted
+    by 'child_word' so we can use binary search to locate the entry. We have to
+    do some fancy bit-work to avoid having to allocate an LmState if a given
+    N-gram is a leaf. We design the child_info in the children_ vector as
+    follows:
+    1. If it's an even number, then it represents a float (i.e. we
+       reinterpret_cast to float), and the associated N-gram is a leaf. This
+       requires losing the least significant bit of information in the float.
+    2. If it's an odd number, then it will be used to represent a pointer to the
+       LmState of the child. In order to use a 32-bit number to represent a
+       possibly 64-bit pointer, we store the LmState structures in memory in
+       a way that's sorted lexicographically by the vector of words, so that
+       following "A B" will be the LmStates for "A B A", "A B B", "A B C" and so
+       on (note, we actually deal with integers instead of letters). So if we
+       make the pointers relative to the current LmState, most of them will be
+       quite small (and all will be positive, due to the lexicographic sorting).
+       As for the pointers that are too large, if any, we can have an "overflow
+       buffer" indexed by a 30-bit index that stores, directly as pointers, the
+       child LmStates. We use the first bit to distinguish the relative pointer
+       case and the overflow pointer case, i.e.,
+       a. If (child_info / 2) is positive, then (current_lmstate_pointer +
+          child_info / 2) is the address of the child LmState.
+       b. If (child_info / 2) is negative, then -1 * (child_info / 2) is the
+          index into the overflow buffer which gives the address of the child
+          LmState.
+
+    Note that unigram LM-states are usually frequently accessed, so it makes
+    sense to assign one LmState to each single word even if it would otherwise
+    be "leaf" as defined above. We then can have an array of those unigram
+    LM-states for efficient lookup.
+
+    Also, we define the class LmState just to set up data structure for Arpa
+    LM. In the end, we have a class like the following:
+
+    class ConstArpaLm {
+     public:
+      // Some public functions.
+     private:
+      // Index of largest word-id, plus one; defines end of "unigram_states_"
+      // array.
+      int32 num_words_;
+
+      // Loopup table for pointers of unigrams. The pointer could be NULL, for
+      // example for those words that are in words.txt, but not in the language
+      // model.
+      int32 **unigram_states_;
+
+      // Number of entries in the overflow buffer for pointers that couldn't be
+      // represented as a 30-bit relative index
+      int32 overflow_buffer_size_;
+
+      // Technically a 32-bit number cannot represent a possibly 64-bit pointer.
+      // We therefore use "relative" address instead of "absolute" address,
+      // which will be a small number most of the time. This buffer is for the
+      // case where the relative address has more than 30-bits.
+      int32 **overflow_buffer_;
+
+      // Size of the array lm_states_. This is required only for I/O.
+      int64 lm_states_size_;
+
+      // Data block for LmState.
+      int32 *lm_states_;
+    };
+
+    Note, when we do I/O, we don't write out the arrays of pointers
+    "overflow_buffer_" and "unigram_states_" directly. Instead we subtract
+    "lm_states_" from each one before writing them out, so we are writing out
+    indexes. Then, when we read them back in, after we allocate "lm_states_"
+    we can convert them back to pointers. When we create these temporary arrays
+    of indexes while reading and writing, we use int64, even if the pointer type
+    of the machine is int32. This way the I/O is independent of the pointer size
+    of the machine.
+
+    Now it is time to put things together.
+
+    ConstArpaLmBuilder takes charge of reading in the Arpa LM and building the
+    ConstArpaLm.
+
+    ConstArpaLM holds the Arpa LM in memory, and provides interfaces for LM
+    operations, such as GetNgramLogprob().
+
+    LmState is an auxiliary class that computes the relative pointers for
+    ConstArpaLmBuilder and ConstArpaLm. It will only be called once during the
+    building process, so it doesn't have to be very efficient.
+
+    In summary, the general building process is as follows:
+    1. In ConstArpaLmBuilder, read in the Arpa format LM. While reading, we keep
+       in memory something like this:
+         std::unordered_map<std::vector<int32>,
+                            LmState*, VectorHasher<int32> > seq_to_state_;
+       The map helps us to convert n-gram entries into LmState (including
+       setting up the parent-children relationship, see above about LmState).
+       Note that at this stage, we don't work on the relative pointers yet.
+    2. In ConstArpaLmBuilder, create a sorted vector from <seq_to_state_>
+         std::vector<std::pair<std::vector<int32>*, LmState*> > sorted_vec;
+       Note, only LmState with non-zero MemSize() should be put into the sorted
+       vector, and we sort it lexicographically according to the word.
+    3. In ConstArpaLmBuilder, update the address for each LmState, relative to
+       the first LmState in the sorted vector (i.e. assume the first LmState has
+       address 0, and work out the rest LmState address using the MemSize() of
+       each LmState).
+    4. In ConstArpaLmBuilder, create a memory block for all the LmStates (after
+       sorting and updating the address). This includes <lm_state_> that stores
+       all the LmStates in an int32 array, <unigram_states_> that keeps the
+       address of unigram LmStates, <overflow_buffer_> that keeps the address
+       of LmState whose address differs too much from the parent address. See
+       above how we handle the leaf case.
+    5. With the information in step 4, create the class ConstArpaLm.
+*/
+
+// Forward declaration of Auxiliary struct ArpaLine.
+struct ArpaLine;
+
+union Int32AndFloat {
+  int32 i;
+  float f;
+
+  Int32AndFloat() {}
+  Int32AndFloat(int32 input_i) : i(input_i) {}
+  Int32AndFloat(float input_f) : f(input_f) {}
+};
+
+class ConstArpaLm {
+ public:
+
+  // Default constructor, will be used if you are going to load the ConstArpaLm
+  // format language model from disk.
+  ConstArpaLm() {
+    lm_states_ = NULL;
+    unigram_states_ = NULL;
+    overflow_buffer_ = NULL;
+    memory_assigned_ = false;
+    initialized_ = false;
+    ngram_order_ = 0;
+  }
+
+  // Special constructor, will be used when you initialize ConstArpaLm from
+  // scratch through this constructor.
+  ConstArpaLm(const int32 bos_symbol, const int32 eos_symbol,
+              const int32 unk_symbol, const int32 ngram_order,
+              const int32 num_words, const int32 overflow_buffer_size,
+              const int64 lm_states_size, int32** unigram_states,
+              int32** overflow_buffer, int32* lm_states) :
+      bos_symbol_(bos_symbol), eos_symbol_(eos_symbol),
+      unk_symbol_(unk_symbol), ngram_order_(ngram_order),
+      num_words_(num_words), overflow_buffer_size_(overflow_buffer_size),
+      lm_states_size_(lm_states_size), unigram_states_(unigram_states),
+      overflow_buffer_(overflow_buffer), lm_states_(lm_states) {
+    KALDI_ASSERT(unigram_states_ != NULL);
+    KALDI_ASSERT(overflow_buffer_ != NULL);
+    KALDI_ASSERT(lm_states_ != NULL);
+    KALDI_ASSERT(ngram_order_ > 0);
+    KALDI_ASSERT(bos_symbol_ < num_words_ && bos_symbol_ > 0);
+    KALDI_ASSERT(eos_symbol_ < num_words_ && eos_symbol_ > 0);
+    KALDI_ASSERT(unk_symbol_ < num_words_ &&
+                 (unk_symbol_ > 0 || unk_symbol_ == -1));
+    lm_states_end_ = lm_states_ + lm_states_size_ - 1;
+    memory_assigned_ = false;
+    initialized_ = true;
+  }
+
+  ~ConstArpaLm() {
+    if (memory_assigned_) {
+      delete[] lm_states_;
+      delete[] unigram_states_;
+      delete[] overflow_buffer_;
+    }
+  }
+
+  // Reads the ConstArpaLm format language model. It calls ReadInternal() or
+  // ReadInternalOldFormat() to do the actual reading.
+  void Read(std::istream &is, bool binary);
+
+  // Writes the language model in ConstArpaLm format.
+  void Write(std::ostream &os, bool binary) const;
+
+  // Creates Arpa format language model from ConstArpaLm format, and writes it
+  // to output stream. This will be useful in testing.
+  void WriteArpa(std::ostream &os) const;
+
+  // Wrapper of GetNgramLogprobRecurse. It first maps possible out-of-vocabulary
+  // words to <unk>, if <unk> is defined, and then calls GetNgramLogprobRecurse.
+  float GetNgramLogprob(const int32 word, const std::vector<int32>& hist) const;
+
+  // Returns true if the history word sequence <hist> has successor, which means
+  // <hist> will be a state in the FST format language model.
+  bool HistoryStateExists(const std::vector<int32>& hist) const;
+
+  int32 BosSymbol() const { return bos_symbol_; }
+  int32 EosSymbol() const { return eos_symbol_; }
+  int32 UnkSymbol() const { return unk_symbol_; }
+  int32 NgramOrder() const { return ngram_order_; }
+  bool Initialized() const { return initialized_; }
+
+ private:
+  // Function that loads data from stream to the class.
+  void ReadInternal(std::istream &is, bool binary);
+
+  // Function that loads data from stream to the class. This is a deprecated one
+  // that handles the old on-disk format. We keep this for back-compatibility
+  // purpose. We have modified the Write() function so for all the new on-disk
+  // format, ReadInternal() will be called.
+  void ReadInternalOldFormat(std::istream &is, bool binary);
+
+  // Loops up n-gram probability for given word sequence. Backoff is handled by
+  // recursively calling this function.
+  float GetNgramLogprobRecurse(const int32 word,
+                               const std::vector<int32>& hist) const;
+
+  // Given a word sequence, find the address of the corresponding LmState.
+  // Returns NULL if no corresponding LmState is found.
+  //
+  // If the word sequence exists in n-gram language model, but it is a leaf and
+  // is not an unigram, we still return NULL, since there is no LmState struct
+  // reserved for this sequence.
+  int32* GetLmState(const std::vector<int32>& seq) const;
+
+  // Given a pointer to the parent, find the child_info that corresponds to
+  // given word. The parent has the following structure:
+  // struct LmState {
+  //   float logprob;
+  //   float backoff_logprob;
+  //   int32 num_children;
+  //   std::pair<int32, int32> [] children;
+  // }
+  // It returns false if the child is not found.
+  bool GetChildInfo(const int32 word, int32* parent, int32* child_info) const;
+
+  // Decodes <child_info> to get log probability and child LmState. In the leaf
+  // case, only <logprob> will be returned, and <child_address> will be NULL.
+  void DecodeChildInfo(const int32 child_info, int32* parent,
+                       int32** child_lm_state, float* logprob) const;
+
+  void WriteArpaRecurse(int32* lm_state,
+                        const std::vector<int32>& seq,
+                        std::vector<ArpaLine> *output) const;
+
+  // We assign memory in Read(). If it is called, we have to release memory in
+  // the destructor.
+  bool memory_assigned_;
+
+  // Makes sure that the language model has been loaded before using it.
+  bool initialized_;
+
+  // Integer corresponds to <s>.
+  int32 bos_symbol_;
+
+  // Integer corresponds to </s>.
+  int32 eos_symbol_;
+
+  // Integer corresponds to unknown-word. -1 if no unknown-word symbol is
+  // provided.
+  int32 unk_symbol_;
+
+  // N-gram order of the language model.
+  int32 ngram_order_;
+
+  // Index of largest word-id plus one. It defines the end of <unigram_states_>
+  // array.
+  int32 num_words_;
+
+  // Number of entries in the overflow buffer for pointers that couldn't be
+  // represented as a 30-bit relative index.
+  int32 overflow_buffer_size_;
+
+  // Size of the <lm_states_> array, which will be needed by I/O.
+  int64 lm_states_size_;
+
+  // Points to the end of <lm_states_>. We use this information to check if
+  // there is any illegal visit to the un-reserved memory.
+  int32* lm_states_end_;
+
+  // Loopup table for pointers of unigrams. The pointer could be NULL, for
+  // example for those words that are in words.txt, but not in the language
+  // model.
+  int32** unigram_states_;
+
+  // Technically a 32-bit number cannot represent a possibly 64-bit pointer. We
+  // therefore use "relative" address instead of "absolute" address, which will
+  // be a small number most of the time. This buffer is for the case where the
+  // relative address has more than 30-bits.
+  int32** overflow_buffer_;
+
+  // Memory chunk that contains the actual LmStates. One LmState has the
+  // following structure:
+  //
+  // struct LmState {
+  //   float logprob;
+  //   float backoff_logprob;
+  //   int32 num_children;
+  //   std::pair<int32, int32> [] children;
+  // }
+  //
+  // Note that the floating point representation has 4 bytes, int32 also has 4
+  // bytes, therefore one LmState will occupy the following number of bytes:
+  //
+  // x = 1 + 1 + 1 + 2 * children.size() = 3 + 2 * children.size()
+  int32* lm_states_;
+};
+
+/**
+ This class wraps a ConstArpaLm format language model with the interface defined
+ in DeterministicOnDemandFst.
+ */
+class ConstArpaLmDeterministicFst
+  : public fst::DeterministicOnDemandFst<fst::StdArc> {
+ public:
+  typedef fst::StdArc::Weight Weight;
+  typedef fst::StdArc::StateId StateId;
+  typedef fst::StdArc::Label Label;
+
+  explicit ConstArpaLmDeterministicFst(const ConstArpaLm& lm);
+
+  // We cannot use "const" because the pure virtual function in the interface is
+  // not const.
+  virtual StateId Start() { return start_state_; }
+
+  // We cannot use "const" because the pure virtual function in the interface is
+  // not const.
+  virtual Weight Final(StateId s);
+
+  virtual bool GetArc(StateId s, Label ilabel, fst::StdArc* oarc);
+
+ private:
+  typedef unordered_map<std::vector<Label>,
+                        StateId, VectorHasher<Label> > MapType;
+  StateId start_state_;
+  MapType wseq_to_state_;
+  std::vector<std::vector<Label> > state_to_wseq_;
+  const ConstArpaLm& lm_;
+};
+
+// Reads in an Arpa format language model and converts it into ConstArpaLm
+// format. We assume that the words in the input Arpa format language model have
+// been converted into integers.
+bool BuildConstArpaLm(const ArpaParseOptions& options,
+                      const std::string& arpa_rxfilename,
+                      const std::string& const_arpa_wxfilename);
+
+}  // namespace kaldi
+
+#endif  // KALDI_LM_CONST_ARPA_LM_H_

runtime/onnxruntime/third_party/kaldi/lm/examples/CREATURE.txt

@@ -0,0 +1,17 @@
+boy
+boy
+boy
+boy
+boy
+boy
+boy
+girl
+girl
+pretty girl
+cat
+dog
+dog
+dog
+rat
+red faced clown
+green caterpillar

runtime/onnxruntime/third_party/kaldi/lm/examples/DAYOFMONTH.txt

@@ -0,0 +1,5 @@
+first
+second
+third
+fourth
+fifth

runtime/onnxruntime/third_party/kaldi/lm/examples/MONTH.txt

@@ -0,0 +1,5 @@
+January
+February
+March
+April
+May

runtime/onnxruntime/third_party/kaldi/lm/examples/YEAR.txt

@@ -0,0 +1,4 @@
+two thousand and ten
+two zero one zero
+two thousand and nine
+two zero zero nine

runtime/onnxruntime/third_party/kaldi/lm/examples/YEARDATE.txt

@@ -0,0 +1 @@
+%#MONTH#% %#DAYOFMONTH#% %#YEAR#%

runtime/onnxruntime/third_party/kaldi/lm/examples/input3.txt

@@ -0,0 +1 @@
+<s> the %#CREATURE#% crossed the street </s>

runtime/onnxruntime/third_party/kaldi/lm/examples/input4.txt

@@ -0,0 +1 @@
+<s> He came here on %#YEARDATE#% </s>

runtime/onnxruntime/third_party/kaldi/lm/irstlm.mk

@@ -0,0 +1,7 @@
+# Additionnal definitions needed to make with IRSTLM toolkit
+
+# Assumes IRSTLM includes and libraries have been installed under 
+#   $(SRCDIR)/../lmtoolkit/include/irstlm and $(SRCDIR)/../lmtoolkit/lib/irstlm
+
+EXTRA_CXXFLAGS = -DHAVE_IRSTLM -I$(SRCDIR)/../lmtoolkit/include -Wno-sign-compare
+EXTRA_LDLIBS = $(SRCDIR)/../lmtoolkit/lib/irstlm/x86_64-apple-darwin10.0/libirstlm.a -lz

runtime/onnxruntime/third_party/kaldi/lm/kaldi-rnnlm.cc

@@ -0,0 +1,140 @@
+// lm/kaldi-rnnlm.cc
+
+// Copyright 2015  Guoguo Chen
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+#include <utility>
+
+#include "lm/kaldi-rnnlm.h"
+#include "util/stl-utils.h"
+#include "util/text-utils.h"
+
+namespace kaldi {
+
+KaldiRnnlmWrapper::KaldiRnnlmWrapper(
+    const KaldiRnnlmWrapperOpts &opts,
+    const std::string &unk_prob_rspecifier,
+    const std::string &word_symbol_table_rxfilename,
+    const std::string &rnnlm_rxfilename) {
+  rnnlm_.setRnnLMFile(rnnlm_rxfilename);
+  rnnlm_.setRandSeed(1);
+  rnnlm_.setUnkSym(opts.unk_symbol);
+  rnnlm_.setUnkPenalty(unk_prob_rspecifier);
+  rnnlm_.restoreNet();
+
+  // Reads symbol table.
+  fst::SymbolTable *word_symbols = NULL;
+  if (!(word_symbols =
+        fst::SymbolTable::ReadText(word_symbol_table_rxfilename))) {
+    KALDI_ERR << "Could not read symbol table from file "
+        << word_symbol_table_rxfilename;
+  }
+  label_to_word_.resize(word_symbols->NumSymbols() + 1);
+  for (int32 i = 0; i < label_to_word_.size() - 1; ++i) {
+    label_to_word_[i] = word_symbols->Find(i);
+    if (label_to_word_[i] == "") {
+      KALDI_ERR << "Could not find word for integer " << i << "in the word "
+          << "symbol table, mismatched symbol table or you have discontinuous "
+          << "integers in your symbol table?";
+    }
+  }
+  label_to_word_[label_to_word_.size() - 1] = opts.eos_symbol;
+  eos_ = label_to_word_.size() - 1;
+}
+
+BaseFloat KaldiRnnlmWrapper::GetLogProb(
+    int32 word, const std::vector<int32> &wseq,
+    const std::vector<float> &context_in,
+    std::vector<float> *context_out) {
+
+  std::vector<std::string> wseq_symbols(wseq.size());
+  for (int32 i = 0; i < wseq_symbols.size(); ++i) {
+    KALDI_ASSERT(wseq[i] < label_to_word_.size());
+    wseq_symbols[i] = label_to_word_[wseq[i]];
+  }
+
+  return rnnlm_.computeConditionalLogprob(label_to_word_[word], wseq_symbols,
+                                          context_in, context_out);
+}
+
+RnnlmDeterministicFst::RnnlmDeterministicFst(int32 max_ngram_order,
+                                             KaldiRnnlmWrapper *rnnlm) {
+  KALDI_ASSERT(rnnlm != NULL);
+  max_ngram_order_ = max_ngram_order;
+  rnnlm_ = rnnlm;
+
+  // Uses empty history for <s>.
+  std::vector<Label> bos;
+  std::vector<float> bos_context(rnnlm->GetHiddenLayerSize(), 1.0);
+  state_to_wseq_.push_back(bos);
+  state_to_context_.push_back(bos_context);
+  wseq_to_state_[bos] = 0;
+  start_state_ = 0;
+}
+
+fst::StdArc::Weight RnnlmDeterministicFst::Final(StateId s) {
+  // At this point, we should have created the state.
+  KALDI_ASSERT(static_cast<size_t>(s) < state_to_wseq_.size());
+
+  std::vector<Label> wseq = state_to_wseq_[s];
+  BaseFloat logprob = rnnlm_->GetLogProb(rnnlm_->GetEos(), wseq,
+                                         state_to_context_[s], NULL);
+  return Weight(-logprob);
+}
+
+bool RnnlmDeterministicFst::GetArc(StateId s, Label ilabel, fst::StdArc *oarc) {
+  // At this point, we should have created the state.
+  KALDI_ASSERT(static_cast<size_t>(s) < state_to_wseq_.size());
+
+  std::vector<Label> wseq = state_to_wseq_[s];
+  std::vector<float> new_context(rnnlm_->GetHiddenLayerSize());
+  BaseFloat logprob = rnnlm_->GetLogProb(ilabel, wseq,
+                                         state_to_context_[s], &new_context);
+
+  wseq.push_back(ilabel);
+  if (max_ngram_order_ > 0) {
+    while (wseq.size() >= max_ngram_order_) {
+      // History state has at most <max_ngram_order_> - 1 words in the state.
+      wseq.erase(wseq.begin(), wseq.begin() + 1);
+    }
+  }
+
+  std::pair<const std::vector<Label>, StateId> wseq_state_pair(
+      wseq, static_cast<Label>(state_to_wseq_.size()));
+
+  // Attemps to insert the current <lseq_state_pair>. If the pair already exists
+  // then it returns false.
+  typedef MapType::iterator IterType;
+  std::pair<IterType, bool> result = wseq_to_state_.insert(wseq_state_pair);
+
+  // If the pair was just inserted, then also add it to <state_to_wseq_> and
+  // <state_to_context_>.
+  if (result.second == true) {
+    state_to_wseq_.push_back(wseq);
+    state_to_context_.push_back(new_context);
+  }
+
+  // Creates the arc.
+  oarc->ilabel = ilabel;
+  oarc->olabel = ilabel;
+  oarc->nextstate = result.first->second;
+  oarc->weight = Weight(-logprob);
+
+  return true;
+}
+
+}  // namespace kaldi

runtime/onnxruntime/third_party/kaldi/lm/kaldi-rnnlm.h

@@ -0,0 +1,104 @@
+// lm/kaldi-rnnlm.h
+
+// Copyright 2015  Guoguo Chen
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef KALDI_LM_KALDI_RNNLM_H_
+#define KALDI_LM_KALDI_RNNLM_H_
+
+#include <string>
+#include <vector>
+
+#include "base/kaldi-common.h"
+#include "fstext/deterministic-fst.h"
+#include "lm/mikolov-rnnlm-lib.h"
+#include "util/common-utils.h"
+
+namespace kaldi {
+
+struct KaldiRnnlmWrapperOpts {
+  std::string unk_symbol;
+  std::string eos_symbol;
+
+  KaldiRnnlmWrapperOpts() : unk_symbol("<RNN_UNK>"), eos_symbol("</s>") {}
+
+  void Register(OptionsItf *opts) {
+    opts->Register("unk-symbol", &unk_symbol, "Symbol for out-of-vocabulary "
+                   "words in rnnlm.");
+    opts->Register("eos-symbol", &eos_symbol, "End of sentence symbol in "
+                   "rnnlm.");
+  }
+};
+
+class KaldiRnnlmWrapper {
+ public:
+  KaldiRnnlmWrapper(const KaldiRnnlmWrapperOpts &opts,
+                    const std::string &unk_prob_rspecifier,
+                    const std::string &word_symbol_table_rxfilename,
+                    const std::string &rnnlm_rxfilename);
+
+  int32 GetHiddenLayerSize() const { return rnnlm_.getHiddenLayerSize(); }
+
+  int32 GetEos() const { return eos_; }
+
+  BaseFloat GetLogProb(int32 word, const std::vector<int32> &wseq,
+                       const std::vector<float> &context_in,
+                       std::vector<float> *context_out);
+
+ private:
+  rnnlm::CRnnLM rnnlm_;
+  std::vector<std::string> label_to_word_;
+  int32 eos_;
+
+  KALDI_DISALLOW_COPY_AND_ASSIGN(KaldiRnnlmWrapper);
+};
+
+class RnnlmDeterministicFst
+    : public fst::DeterministicOnDemandFst<fst::StdArc> {
+ public:
+  typedef fst::StdArc::Weight Weight;
+  typedef fst::StdArc::StateId StateId;
+  typedef fst::StdArc::Label Label;
+
+  // Does not take ownership.
+  RnnlmDeterministicFst(int32 max_ngram_order, KaldiRnnlmWrapper *rnnlm);
+
+  // We cannot use "const" because the pure virtual function in the interface is
+  // not const.
+  virtual StateId Start() { return start_state_; }
+
+  // We cannot use "const" because the pure virtual function in the interface is
+  // not const.
+  virtual Weight Final(StateId s);
+
+  virtual bool GetArc(StateId s, Label ilabel, fst::StdArc* oarc);
+
+ private:
+  typedef unordered_map<std::vector<Label>,
+                        StateId, VectorHasher<Label> > MapType;
+  StateId start_state_;
+  MapType wseq_to_state_;
+  std::vector<std::vector<Label> > state_to_wseq_;
+
+  KaldiRnnlmWrapper *rnnlm_;
+  int32 max_ngram_order_;
+  std::vector<std::vector<float> > state_to_context_;
+};
+
+}  // namespace kaldi
+
+#endif  // KALDI_LM_KALDI_RNNLM_H_

runtime/onnxruntime/third_party/kaldi/lm/kenlm-test.cc

@@ -0,0 +1,84 @@
+#ifdef HAVE_KENLM
+#include "util/text-utils.h"
+#include "util/kaldi-io.h"
+#include "lm/kenlm.h"
+
+namespace kaldi {
+
+void UnitTestKenLm() {
+  // construct symbol_to_symbol_id to map word string to kaldi symbol index,
+  // in practice, hypothesis is already intergerized, no need for this mapping.
+  std::unordered_map<std::string, int32> symbol_to_symbol_id;
+  std::ifstream symbol_table_stream("test_data/words.txt");
+  std::string line;
+  while (std::getline(symbol_table_stream, line)) {
+    std::vector<std::string> fields;
+    SplitStringToVector(line, " ", true, &fields);
+    if (fields.size() == 2) {
+      std::string symbol = fields[0];
+      uint32 symbol_id = -1;
+      ConvertStringToInteger(fields[1], &symbol_id);
+      symbol_to_symbol_id[symbol] = symbol_id;
+    }
+  }
+
+  // open testing stream, one sentence per line, in raw text form
+  std::ifstream is("test_data/sentences.txt");
+
+  KenLm lm;
+  lm.Load("test_data/lm.kenlm", "test_data/words.txt");
+  KenLmDeterministicOnDemandFst<fst::StdArc> lm_fst(&lm);
+
+  std::string sentence;
+  while(std::getline(is, sentence)) {
+    std::vector<std::string> words;
+    SplitStringToVector(sentence, " ", true, &words);
+    words.push_back("</s>");
+
+    // 1. test KenLm interface: this is only for test purpose,
+    //    you should not use kenlm this way in Kaldi.
+    std::string sentence_log = "[KENLM]";
+
+    KenLm::State state[2];
+    KenLm::State* istate = &state[0];
+    KenLm::State* ostate = &state[1];
+
+    lm.SetStateToBeginOfSentence(istate);
+
+    for (int i = 0; i < words.size(); i++) {
+      std::string word = words[i];
+      BaseFloat log10_word_score = lm.Score(istate, lm.GetWordIndex(word), ostate);
+      sentence_log += " " + word + 
+                      "[" + std::to_string(lm.GetWordIndex(word)) + "]=" + 
+                      std::to_string(-log10_word_score * M_LN10); //convert to -ln()
+      std::swap(istate, ostate);
+    }
+    KALDI_LOG << sentence_log;
+
+    // 2. test Fst wrapper interface (KenLmDeterministicFst),
+    //    this is the recommanded way to interact with Kaldi's Fst framework.
+    sentence_log = "(KALDI)";
+    KenLmDeterministicOnDemandFst<fst::StdArc>::StateId s = lm_fst.Start();
+    for (int i = 0; i < words.size(); i++) {
+      int32 symbol_id = symbol_to_symbol_id[words[i]];
+      sentence_log += " " + words[i] + "(" + std::to_string(symbol_id) + ")=";
+      if (words[i] == "</s>") {
+        sentence_log += std::to_string(lm_fst.Final(s).Value());
+      } else {
+        fst::StdArc arc;
+        lm_fst.GetArc(s, symbol_id, &arc);
+        s = arc.nextstate;
+        sentence_log += std::to_string(arc.weight.Value());
+      }
+    }
+    KALDI_LOG << sentence_log;
+  }
+}
+} // namespace kaldi
+
+int main(int argc, char *argv[]) {
+  using namespace kaldi;
+  UnitTestKenLm();
+  return 0;
+}
+#endif

runtime/onnxruntime/third_party/kaldi/lm/kenlm.cc

@@ -0,0 +1,99 @@
+// Copyright 2020  Jiayu DU
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+#ifdef HAVE_KENLM
+#include "lm/kenlm.h"
+
+namespace kaldi {
+
+void KenLm::ComputeSymbolToWordIndexMapping(std::string symbol_table_filename) {
+  // count symbol table size
+  int num_syms = 0;
+  std::string line;
+  std::ifstream is(symbol_table_filename);
+  while(std::getline(is, line)) {
+    if (!line.empty()) num_syms++;
+  }
+
+  symid_to_wid_.clear();
+  symid_to_wid_.resize(num_syms, 0);
+
+  is.clear();
+  is.seekg(0);
+  int num_mapped = 0;
+  while (std::getline(is, line)) {
+    std::vector<std::string> fields;
+    SplitStringToVector(line, " ", true, &fields);
+    if (fields.size() == 2) {
+      std::string sym = fields[0];
+      int32 symid = 0;  ConvertStringToInteger(fields[1], &symid);
+      // mark special LM word
+      if (sym == bos_sym_) {
+        bos_symid_ = symid;
+      } else if (sym == eos_sym_) {
+        eos_symid_ = symid;
+      } else if (sym == "<unk>" || sym == "<UNK>") {
+        unk_sym_ = sym;
+        unk_symid_ = symid;
+      }
+      // check vocabulary consistency between kaldi and kenlm.
+      // note we always handle <unk> & <UNK> as a pair, 
+      // so don't worry about the literal mismatch
+      // between Kaldi and kenlm arpa (<UNK> vs <unk>)
+      WordIndex wid = vocab_->Index(sym.c_str());
+      if ((wid == vocab_->Index("<unk>") || wid == vocab_->Index("<UNK>")) 
+          && sym != "<unk>" && sym != "<UNK>"
+          && sym != "<eps>"
+          && sym != "#0") {
+        KALDI_ERR << "found mismatched symbol: " << sym
+                  << ", this symbol is in Kaldi, but is unseen in KenLm"
+                  << ", they should have strictly consistent vocabulary.";
+      } else {
+        symid_to_wid_[symid] = wid;
+        num_mapped += 1;
+      }
+    }
+  }
+  KALDI_ASSERT(num_mapped == symid_to_wid_.size());
+  KALDI_LOG << "Successfully mapped " << num_mapped 
+            << " Kaldi symbols to KenLm words";
+}
+
+int KenLm::Load(std::string kenlm_filename,
+                std::string symbol_table_filename,
+                util::LoadMethod load_method) {
+  if (model_ != nullptr) { delete model_; }
+  model_ = nullptr;
+  vocab_ = nullptr;
+
+  // load KenLm model
+  lm::ngram::Config config;
+  config.load_method = load_method;
+  model_ = lm::ngram::LoadVirtual(kenlm_filename.c_str(), config);
+  if (model_ == nullptr) { KALDI_ERR << "Failed to load KenLm model"; }
+
+  // KenLm holds vocabulary internally with ownership,
+  // vocab_ here is just for concise reference
+  vocab_ = &model_->BaseVocabulary();
+  if (vocab_ == nullptr) { KALDI_ERR << "Failed to get vocabulary from KenLm model"; }
+
+  // compute the index mapping from Kaldi symbol to KenLm word
+  ComputeSymbolToWordIndexMapping(symbol_table_filename);
+  return 0;
+}
+
+} // namespace kaldi
+#endif

runtime/onnxruntime/third_party/kaldi/lm/kenlm.h

@@ -0,0 +1,207 @@
+// Copyright 2020  Jiayu DU
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+#ifdef HAVE_KENLM
+#ifndef KALDI_LM_KENLM_H
+#define KALDI_LM_KENLM_H
+
+#include <base/kaldi-common.h>
+#include <fst/fstlib.h>
+#include <fst/fst-decl.h>
+#include <fstext/deterministic-fst.h>
+
+#include "lm/model.hh"
+#include "util/murmur_hash.hh"
+
+namespace kaldi {
+
+// KenLm class wraps kenlm model(supporting both "trie" or "probing" models):
+//  1. provides interface for loading binary LM, and holds it with ownership
+//  2. provides interface for ngram score query at runtime
+//  3. handles the index mapping between kaldi's symbols & kenlm's words
+// KenLm object is heavy, stateless and thread-safe, 
+// can be shared by Fst wrapper class(i.e. KenLmDeterministicOnDemandFst)
+class KenLm {
+ public:
+  typedef lm::WordIndex WordIndex;
+  typedef lm::ngram::State State;
+
+ public:
+  KenLm() : 
+    model_(nullptr), vocab_(nullptr),
+    bos_sym_("<s>"), eos_sym_("</s>"), unk_sym_("<unk>"),
+    bos_symid_(0), eos_symid_(0), unk_symid_(0)
+  { }
+
+  ~KenLm() {
+    if (model_ != nullptr) {
+      delete model_;
+    }
+    model_ = nullptr;
+    vocab_ = nullptr;
+    symid_to_wid_.clear();
+  }
+
+  // If you have big LM on SSD hard-drive,
+  // you can set load_method to util::LoadMethod::LAZY,
+  // which enables "on-demand" model reading(via POSIX mmap) at runtime.
+  // Refer to tools/kenlm/util/mmap.hh for more load methods.
+  int Load(std::string kenlm_filename, 
+           std::string kaldi_symbol_table_filename,
+           util::LoadMethod load_method = util::LoadMethod::POPULATE_OR_READ);
+
+  inline WordIndex GetWordIndex(std::string word) const {
+    return vocab_->Index(word.c_str());
+  }
+
+  inline WordIndex GetWordIndex(int32 symbol_id) const {
+    return symid_to_wid_[symbol_id];
+  }
+
+  void SetStateToBeginOfSentence(State *s) const { model_->BeginSentenceWrite(s); }
+  void SetStateToNull(State *s) const { model_->NullContextWrite(s); }
+
+  int32 BosSymbolIndex() const { return bos_symid_; }
+  int32 EosSymbolIndex() const { return eos_symid_; }
+  int32 UnkSymbolIndex() const { return unk_symid_; }
+
+  inline BaseFloat Score(const State *in_state,
+                         WordIndex word,
+                         State *out_state) const {
+    return model_->BaseScore(in_state, word, out_state);
+  }
+
+  // This provides a fast state hashing, 
+  // KenLmDeterministicOnDemandFst needs this for Fst states managing.
+  struct StateHasher {
+    inline size_t operator()(const State &s) const noexcept {
+      return util::MurmurHashNative(s.words, sizeof(WordIndex) * s.Length());
+    }
+  };
+
+ private:
+  void ComputeSymbolToWordIndexMapping(std::string symbol_table);
+  
+ private:
+  lm::base::Model *model_; // with ownership
+
+  // without ownership, points to internal vocabulary of model_
+  const lm::base::Vocabulary* vocab_;
+
+  // There are two integerized indexing systems here:
+  // 1. Kaldi's fst output *symbol index*(defined in words.txt),
+  // 2. KenLm's *word index*(defined by word string hashing).
+  // In order to rescore kaldi hypotheses with kenlm ngrams, 
+  // we need to know the index mapping from symbol to word.
+  // KenLm class precomputes (during model loading) and stores this mapping,
+  // and apply the mapping at runtime.
+  // This is slower, but at least we don't need
+  // to modify/convert runtime resources.(e.g. HCLG/lattices or kenlm models)
+  //
+  // In the mapping, <eps> and #0 symbols are special:
+  // They do not correspond to any word in KenLm,
+  // so the mapping of these two symbols are logically undefined,
+  // we just map them to KenLm's <unk> to avoid random invalid mapping.
+
+  // symid_to_wid_[kaldi_symbol_index] -> kenlm word index
+  std::vector<WordIndex> symid_to_wid_;
+
+  // special lm symbols
+  std::string bos_sym_;
+  std::string eos_sym_;
+  std::string unk_sym_;
+
+  int32 bos_symid_;
+  int32 eos_symid_;
+  int32 unk_symid_;
+}; // class KenLm
+
+
+// DeterministicOnDemandFst wraps a KenLm object as a deteministic Fst.
+// Internally, it manages dynamically expanded Fst states(so not thread-safe),
+// different threads should create their own instances of this class.
+// They are lightweight and can share the same KenLm object.
+template<class Arc>
+class KenLmDeterministicOnDemandFst : public fst::DeterministicOnDemandFst<Arc> {
+ public:
+  typedef typename Arc::Weight Weight;
+  typedef typename Arc::StateId StateId;
+  typedef typename Arc::Label Label;
+  typedef typename KenLm::State State;
+  typedef typename KenLm::WordIndex WordIndex;
+ 
+  explicit KenLmDeterministicOnDemandFst(const KenLm *lm)
+   : lm_(lm), num_states_(0), bos_state_id_(0)
+  {
+    // create bos to be FST start state
+    MapElem e;
+    lm->SetStateToBeginOfSentence(&e.first);
+    e.second = bos_state_id_;
+    std::pair<IterType, bool> r = state_map_.insert(e);
+    KALDI_ASSERT(r.second == true); // bos successfully inserted into state map
+    state_vec_.push_back(&r.first->first);
+    num_states_++;
+
+    eos_symbol_id_ = lm_->EosSymbolIndex();
+  }
+  virtual ~KenLmDeterministicOnDemandFst() { }
+
+  virtual StateId Start() { 
+    return bos_state_id_;
+  }
+
+  virtual bool GetArc(StateId s, Label label, Arc *oarc) {
+    KALDI_ASSERT(s < static_cast<StateId>(state_vec_.size()));
+    const State* istate = state_vec_[s];
+    MapElem e;
+    WordIndex word = lm_->GetWordIndex(label);
+    BaseFloat log_10_prob = lm_->Score(istate, word, &e.first);
+    e.second = num_states_;
+    std::pair<IterType, bool> r = state_map_.insert(e);
+    if (r.second == true) { // new state
+      state_vec_.push_back(&(r.first->first));
+      num_states_++;
+    }
+
+    oarc->ilabel = label;
+    oarc->olabel = oarc->ilabel;
+    oarc->nextstate = r.first->second;
+    oarc->weight = Weight(-log_10_prob * M_LN10); // KenLm log10() -> Kaldi ln()
+
+    return true;
+  }
+
+  virtual Weight Final(StateId s) {
+    Arc oarc;
+    GetArc(s, eos_symbol_id_, &oarc);
+    return oarc.weight;
+  }
+
+ private:
+  typedef std::pair<State, StateId> MapElem;
+  typedef unordered_map<State, StateId, KenLm::StateHasher> MapType;
+  typedef typename MapType::iterator IterType;
+
+  const KenLm *lm_; // no ownership
+  MapType state_map_;
+  std::vector<const State*> state_vec_;
+  StateId num_states_; // state vector index range, [0, num_states_)
+  StateId bos_state_id_;  // fst start state id
+  Label eos_symbol_id_;
+}; // class KenLmDeterministicOnDemandFst
+} // namespace kaldi
+#endif
+#endif

runtime/onnxruntime/third_party/kaldi/lm/mikolov-rnnlm-lib.h

@@ -0,0 +1,239 @@
+// lm/mikolov-rnnlm-lib.h
+
+// Copyright       2015  Guoguo Chen  Hainan Xu
+//            2010-2012  Tomas Mikolov
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// This file is based on version 0.3e of the RNNLM language modeling
+// toolkit by Tomas Mikolov.  Changes made by authors other than
+// Tomas Mikolov are licensed under the Apache License, the short form
+// os which is below.  The original code by Tomas Mikolov is licensed
+// under the BSD 3-clause license, whose text is further below.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+//
+//
+// Original BSD 3-clause license text:
+// Copyright (c) 2010-2012 Tomas Mikolov
+//
+// All rights reserved. Redistribution and use in source and binary forms, with
+// or without modification, are permitted provided that the following conditions
+// are met: 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following
+// disclaimer. 2. Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the
+// distribution. 3. Neither name of copyright holders nor the names of its
+// contributors may be used to endorse or promote products derived from this
+// software without specific prior written permission. THIS SOFTWARE IS PROVIDED
+// BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+// EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+// OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef KALDI_LM_MIKOLOV_RNNLM_LIB_H_
+#define KALDI_LM_MIKOLOV_RNNLM_LIB_H_
+
+#include <string>
+#include <vector>
+#include "util/stl-utils.h"
+
+namespace rnnlm {
+
+#define MAX_STRING 100
+#define MAX_FILENAME_STRING 300
+
+typedef double real;      //  doubles for NN weights
+typedef double direct_t;  //  doubles for ME weights;
+
+struct neuron {
+  real ac;    // actual value stored in neuron
+  real er;    // error value in neuron, used by learning algorithm
+};
+
+struct synapse {
+  real weight;  // weight of synapse
+};
+
+struct vocab_word {
+  int cn;
+  char word[MAX_STRING];
+
+  real prob;
+  int class_index;
+};
+
+const unsigned int PRIMES[] = {108641969, 116049371, 125925907, 133333309,
+  145678979, 175308587, 197530793, 234567803, 251851741, 264197411,
+  330864029, 399999781,
+  407407183, 459258997, 479012069, 545678687, 560493491, 607407037, 629629243,
+  656789717, 716048933, 718518067, 725925469, 733332871, 753085943, 755555077,
+  782715551, 790122953, 812345159, 814814293, 893826581, 923456189, 940740127,
+  953085797, 985184539, 990122807};
+const unsigned int PRIMES_SIZE  =  sizeof(PRIMES) / sizeof(PRIMES[0]);
+
+const int MAX_NGRAM_ORDER = 20;
+
+enum FileTypeEnum {TEXT, BINARY, COMPRESSED};  // COMPRESSED not yet implemented
+
+class CRnnLM {
+ protected:
+  char train_file[MAX_FILENAME_STRING];
+  char valid_file[MAX_FILENAME_STRING];
+  char test_file[MAX_FILENAME_STRING];
+  char rnnlm_file[MAX_FILENAME_STRING];
+  char lmprob_file[MAX_FILENAME_STRING];
+
+  int rand_seed;
+  int version;
+  int filetype;
+
+  int use_lmprob;
+  real gradient_cutoff;
+
+  real dynamic;
+
+  real alpha;
+  real starting_alpha;
+  int alpha_divide;
+  double logp, llogp;
+  float min_improvement;
+  int iter;
+  int vocab_max_size;
+  int vocab_size;
+  int train_words;
+  int train_cur_pos;
+  int counter;
+
+  int anti_k;
+
+  real beta;
+
+  int class_size;
+  int **class_words;
+  int *class_cn;
+  int *class_max_cn;
+  int old_classes;
+
+  struct vocab_word *vocab;
+  void sortVocab();
+  int *vocab_hash;
+  int vocab_hash_size;
+
+  int layer0_size;
+  int layer1_size;
+  int layerc_size;
+  int layer2_size;
+
+  long long direct_size;
+  int direct_order;
+  int history[MAX_NGRAM_ORDER];
+
+  int bptt;
+  int bptt_block;
+  int *bptt_history;
+  neuron *bptt_hidden;
+  struct synapse *bptt_syn0;
+
+  int gen;
+
+  int independent;
+
+  struct neuron *neu0;    // neurons in input layer
+  struct neuron *neu1;    // neurons in hidden layer
+  struct neuron *neuc;    // neurons in hidden layer
+  struct neuron *neu2;    // neurons in output layer
+
+  struct synapse *syn0;   // weights between input and hidden layer
+  struct synapse *syn1;   // weights between hidden and output layer
+                          // (or hidden and compression if compression>0)
+  struct synapse *sync;   // weights between hidden and compression layer
+  direct_t *syn_d;        // direct parameters between input and output layer
+                          // (similar to Maximum Entropy model parameters)
+
+  // backup used in training:
+  struct neuron *neu0b;
+  struct neuron *neu1b;
+  struct neuron *neucb;
+  struct neuron *neu2b;
+
+  struct synapse *syn0b;
+  struct synapse *syn1b;
+  struct synapse *syncb;
+  direct_t *syn_db;
+
+  // backup used in n-bset rescoring:
+  struct neuron *neu1b2;
+
+  unordered_map<std::string, float> unk_penalty;
+  std::string unk_sym;
+
+ public:
+
+  int alpha_set, train_file_set;
+
+  CRnnLM();
+
+  ~CRnnLM();
+
+  real random(real min, real max);
+
+  void setRnnLMFile(const std::string &str);
+  int getHiddenLayerSize() const { return layer1_size; }
+  void setRandSeed(int newSeed);
+
+  int getWordHash(const char *word);
+  void readWord(char *word, FILE *fin);
+  int searchVocab(const char *word);
+
+  void saveWeights();      // saves current weights and unit activations
+  void initNet();
+  void goToDelimiter(int delim, FILE *fi);
+  void restoreNet();
+  void netReset();         // will erase just hidden layer state + bptt history
+                           // + maxent history (called at end of sentences in
+                           // the independent mode)
+
+  void computeNet(int last_word, int word);
+  void copyHiddenLayerToInput();
+
+  void matrixXvector(struct neuron *dest, struct neuron *srcvec,
+                     struct synapse *srcmatrix, int matrix_width,
+                     int from, int to, int from2, int to2, int type);
+
+  void restoreContextFromVector(const std::vector<float> &context_in);
+  void saveContextToVector(std::vector<float> *context_out);
+
+  float computeConditionalLogprob(
+      std::string current_word,
+      const std::vector<std::string> &history_words,
+      const std::vector<float> &context_in,
+      std::vector<float> *context_out);
+
+  void setUnkSym(const std::string &unk);
+  void setUnkPenalty(const std::string &filename);
+  float getUnkPenalty(const std::string &word);
+  bool isUnk(const std::string &word);
+};
+
+}  // namespace rnnlm
+
+#endif  // KALDI_LM_MIKOLOV_RNNLM_LIB_H_

runtime/onnxruntime/third_party/kaldi/lm/test_data/input.arpa

@@ -0,0 +1,21 @@
+
+\data\
+ngram 1=4
+ngram 2=2
+ngram 3=2
+
+\1-grams:
+-5.234679	a -3.3
+-3.456783	b
+0.0000000	<s> -2.5
+-4.333333	</s>
+
+\2-grams:
+-1.45678	a b -3.23
+-1.30490	<s> a -4.2
+
+\3-grams:
+-0.34958	<s> a b
+-0.23940	a b </s>
+
+\end\

runtime/onnxruntime/third_party/kaldi/lm/test_data/missing_backoffs.arpa

@@ -0,0 +1,20 @@
+
+\data\
+ngram 1=4
+ngram 2=1
+ngram 3=2
+
+\1-grams:
+-5.234679	a -3.3
+-3.456783	b
+0.0000000	<s> -2.5
+-4.333333	</s>
+
+\2-grams:
+-1.30490	<s> a -4.2
+
+\3-grams:
+-0.34958	<s> a b
+-0.23940	a b </s>
+
+\end\

runtime/onnxruntime/third_party/kaldi/lm/test_data/missing_bos.arpa

@@ -0,0 +1,18 @@
+
+\data\
+ngram 1=3
+ngram 2=1
+ngram 3=1
+
+\1-grams:
+-5.234679	a -3.3
+-3.456783	b -3.0
+-4.333333	</s>
+
+\2-grams:
+-1.45678	a b -3.23
+
+\3-grams:
+-0.23940	a b </s>
+
+\end\

runtime/onnxruntime/third_party/kaldi/lm/test_data/unused_backoffs.arpa

@@ -0,0 +1,26 @@
+
+\data\
+ngram 1=4
+ngram 2=2
+ngram 3=2
+ngram 4=2
+
+\1-grams:
+-5.234679	a -3.3
+-3.456783	b
+0.0000000	<s> -2.5
+-4.333333	</s>
+
+\2-grams:
+-1.45678	a b -3.23
+-1.30490	<s> a -4.2
+
+\3-grams:
+-0.34958	<s> a b
+-0.23940	a b </s>
+
+\4-grams:
+-0.01888	<s> a b b
+-0.03333        <s> b b b
+
+\end\