Initial commit: FunASR Speech Recognition Toolkit
Update API Documentation / build-api-docs (push) Has been cancelled
Update API Documentation / build-api-docs (push) Has been cancelled
Add complete FunASR codebase including models, runtime, and documentation.
This commit is contained in:
freedakgmail
@@ -0,0 +1,373 @@
|
||||
// lm/arpa-file-parser-test.cc
|
||||
|
||||
// Copyright 2016 Smart Action Company LLC (kkm)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/**
|
||||
* @file arpa-file-parser-test.cc
|
||||
* @brief Unit tests for language model code.
|
||||
*/
|
||||
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
|
||||
#include "base/kaldi-common.h"
|
||||
#include "fst/fstlib.h"
|
||||
#include "lm/arpa-file-parser.h"
|
||||
|
||||
namespace kaldi {
|
||||
namespace {
|
||||
|
||||
const int kMaxOrder = 3;
|
||||
|
||||
struct NGramTestData {
|
||||
int32 line_number;
|
||||
float logprob;
|
||||
int32 words[kMaxOrder];
|
||||
float backoff;
|
||||
};
|
||||
|
||||
std::ostream& operator<<(std::ostream &os, const NGramTestData &data) {
|
||||
std::ios::fmtflags saved_state(os.flags());
|
||||
os << std::fixed << std::setprecision(6);
|
||||
|
||||
os << data.logprob << ' ';
|
||||
for (int i = 0; i < kMaxOrder; ++i) os << data.words[i] << ' ';
|
||||
os << data.backoff << " // Line " << data.line_number;
|
||||
|
||||
os.flags(saved_state);
|
||||
return os;
|
||||
}
|
||||
|
||||
// This does not own the array pointer, and uset to simplify passing expected
|
||||
// result to TestableArpaFileParser::Verify.
|
||||
template <class T>
|
||||
struct CountedArray {
|
||||
template <size_t N>
|
||||
CountedArray(T(&array)[N]) : array(array), count(N) { }
|
||||
const T *array;
|
||||
const size_t count;
|
||||
};
|
||||
|
||||
template <class T, size_t N>
|
||||
inline CountedArray<T> MakeCountedArray(T(&array)[N]) {
|
||||
return CountedArray<T>(array);
|
||||
}
|
||||
|
||||
class TestableArpaFileParser : public ArpaFileParser {
|
||||
public:
|
||||
TestableArpaFileParser(const ArpaParseOptions &options,
|
||||
fst::SymbolTable *symbols)
|
||||
: ArpaFileParser(options, symbols),
|
||||
header_available_(false),
|
||||
read_complete_(false),
|
||||
last_order_(0) { }
|
||||
void Validate(CountedArray<int32> counts, CountedArray<NGramTestData> ngrams);
|
||||
|
||||
private:
|
||||
// ArpaFileParser overrides.
|
||||
virtual void HeaderAvailable();
|
||||
virtual void ConsumeNGram(const NGram& ngram);
|
||||
virtual void ReadComplete();
|
||||
|
||||
bool header_available_;
|
||||
bool read_complete_;
|
||||
int32 last_order_;
|
||||
std::vector<NGramTestData> ngrams_;
|
||||
};
|
||||
|
||||
void TestableArpaFileParser::HeaderAvailable() {
|
||||
KALDI_ASSERT(!header_available_);
|
||||
KALDI_ASSERT(!read_complete_);
|
||||
header_available_ = true;
|
||||
KALDI_ASSERT(NgramCounts().size() <= kMaxOrder);
|
||||
}
|
||||
|
||||
void TestableArpaFileParser::ConsumeNGram(const NGram& ngram) {
|
||||
KALDI_ASSERT(header_available_);
|
||||
KALDI_ASSERT(!read_complete_);
|
||||
KALDI_ASSERT(ngram.words.size() <= NgramCounts().size());
|
||||
KALDI_ASSERT(ngram.words.size() >= last_order_);
|
||||
last_order_ = ngram.words.size();
|
||||
|
||||
NGramTestData entry = { 0 };
|
||||
entry.line_number = LineNumber();
|
||||
entry.logprob = ngram.logprob;
|
||||
entry.backoff = ngram.backoff;
|
||||
std::copy(ngram.words.begin(), ngram.words.end(), entry.words);
|
||||
ngrams_.push_back(entry);
|
||||
}
|
||||
|
||||
void TestableArpaFileParser::ReadComplete() {
|
||||
KALDI_ASSERT(header_available_);
|
||||
KALDI_ASSERT(!read_complete_);
|
||||
read_complete_ = true;
|
||||
}
|
||||
|
||||
bool CompareNgrams(const NGramTestData &actual,
|
||||
NGramTestData expected) {
|
||||
expected.logprob *= Log(10.0);
|
||||
expected.backoff *= Log(10.0);
|
||||
if (actual.line_number != expected.line_number
|
||||
|| !std::equal(actual.words, actual.words + kMaxOrder,
|
||||
expected.words)
|
||||
|| !ApproxEqual(actual.logprob, expected.logprob)
|
||||
|| !ApproxEqual(actual.backoff, expected.backoff)) {
|
||||
KALDI_WARN << "Actual n-gram [" << actual
|
||||
<< "] differs from expected [" << expected << "]";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void TestableArpaFileParser::Validate(
|
||||
CountedArray<int32> expect_counts,
|
||||
CountedArray<NGramTestData> expect_ngrams) {
|
||||
// This needs better disagnostics probably.
|
||||
KALDI_ASSERT(NgramCounts().size() == expect_counts.count);
|
||||
KALDI_ASSERT(std::equal(NgramCounts().begin(), NgramCounts().end(),
|
||||
expect_counts.array));
|
||||
|
||||
KALDI_ASSERT(ngrams_.size() == expect_ngrams.count);
|
||||
// auto mpos = std::mismatch(ngrams_.begin(), ngrams_.end(),
|
||||
// expect_ngrams.array, CompareNgrams);
|
||||
// if (mpos.first != ngrams_.end())
|
||||
// KALDI_ERR << "Maismatch at index " << mpos.first - ngrams_.begin();
|
||||
// TODO: auto above requres C++11, and I cannot spell out the type!!!
|
||||
KALDI_ASSERT(std::equal(ngrams_.begin(), ngrams_.end(),
|
||||
expect_ngrams.array, CompareNgrams));
|
||||
}
|
||||
|
||||
// Read integer LM (no symbols) with log base conversion.
|
||||
void ReadIntegerLmLogconvExpectSuccess() {
|
||||
KALDI_LOG << "ReadIntegerLmLogconvExpectSuccess()";
|
||||
|
||||
static std::string integer_lm = "\
|
||||
\\data\\\n\
|
||||
ngram 1=4\n\
|
||||
ngram 2=2\n\
|
||||
ngram 3=2\n\
|
||||
\n\
|
||||
\\1-grams:\n\
|
||||
-5.2\t4\t-3.3\n\
|
||||
-3.4\t5\n\
|
||||
0\t1\t-2.5\n\
|
||||
-4.3\t2\n\
|
||||
\n\
|
||||
\\2-grams:\n\
|
||||
-1.4\t4 5\t-3.2\n\
|
||||
-1.3\t1 4\t-4.2\n\
|
||||
\n\
|
||||
\\3-grams:\n\
|
||||
-0.3\t1 4 5\n\
|
||||
-0.2\t4 5 2\n\
|
||||
\n\
|
||||
\\end\\";
|
||||
|
||||
int32 expect_counts[] = { 4, 2, 2 };
|
||||
NGramTestData expect_ngrams[] = {
|
||||
{ 7, -5.2, { 4, 0, 0 }, -3.3 },
|
||||
{ 8, -3.4, { 5, 0, 0 }, 0.0 },
|
||||
{ 9, 0.0, { 1, 0, 0 }, -2.5 },
|
||||
{ 10, -4.3, { 2, 0, 0 }, 0.0 },
|
||||
|
||||
{ 13, -1.4, { 4, 5, 0 }, -3.2 },
|
||||
{ 14, -1.3, { 1, 4, 0 }, -4.2 },
|
||||
|
||||
{ 17, -0.3, { 1, 4, 5 }, 0.0 },
|
||||
{ 18, -0.2, { 4, 5, 2 }, 0.0 } };
|
||||
|
||||
ArpaParseOptions options;
|
||||
options.bos_symbol = 1;
|
||||
options.eos_symbol = 2;
|
||||
|
||||
TestableArpaFileParser parser(options, NULL);
|
||||
std::istringstream stm(integer_lm, std::ios_base::in);
|
||||
parser.Read(stm);
|
||||
parser.Validate(MakeCountedArray(expect_counts),
|
||||
MakeCountedArray(expect_ngrams));
|
||||
}
|
||||
|
||||
// \xCE\xB2 = UTF-8 for Greek beta, to churn some UTF-8 cranks.
|
||||
static std::string symbolic_lm = "\
|
||||
We also allow random text coming before the \\data\\\n\
|
||||
section marker. Even this is ok:\n\
|
||||
\n\
|
||||
\\1-grams:\n\
|
||||
\n\
|
||||
and should be ignored before the \\data\\ marker\n\
|
||||
is seen alone by itself on a line.\n\
|
||||
\n\
|
||||
\\data\\\n\
|
||||
ngram 1=4\n\
|
||||
ngram 2=2\n\
|
||||
ngram 3=2\n\
|
||||
\n\
|
||||
\\1-grams: \n\
|
||||
-5.2\ta\t-3.3\n\
|
||||
-3.4\t\xCE\xB2\n\
|
||||
0.0\t<s>\t-2.5\n\
|
||||
-4.3\t</s>\n\
|
||||
\n\
|
||||
\\2-grams:\t\n\
|
||||
-1.5\ta \xCE\xB2\t-3.2\n\
|
||||
-1.3\t<s> a\t-4.2\n\
|
||||
\n\
|
||||
\\3-grams:\n\
|
||||
-0.3\t<s> a \xCE\xB2\n\
|
||||
-0.2\t<s> a </s>\n\
|
||||
\\end\\";
|
||||
|
||||
// Symbol table that is created with predefined test symbols, "a" but no "b".
|
||||
class TestSymbolTable : public fst::SymbolTable {
|
||||
public:
|
||||
TestSymbolTable() {
|
||||
AddSymbol("<eps>", 0);
|
||||
AddSymbol("<s>", 1);
|
||||
AddSymbol("</s>", 2);
|
||||
AddSymbol("<unk>", 3);
|
||||
AddSymbol("a", 4);
|
||||
}
|
||||
};
|
||||
|
||||
// Full expected result shared between ReadSymbolicLmNoOovImpl and
|
||||
// ReadSymbolicLmWithOovAddToSymbols().
|
||||
NGramTestData expect_symbolic_full[] = {
|
||||
{ 15, -5.2, { 4, 0, 0 }, -3.3 },
|
||||
{ 16, -3.4, { 5, 0, 0 }, 0.0 },
|
||||
{ 17, 0.0, { 1, 0, 0 }, -2.5 },
|
||||
{ 18, -4.3, { 2, 0, 0 }, 0.0 },
|
||||
|
||||
{ 21, -1.5, { 4, 5, 0 }, -3.2 },
|
||||
{ 22, -1.3, { 1, 4, 0 }, -4.2 },
|
||||
|
||||
{ 25, -0.3, { 1, 4, 5 }, 0.0 },
|
||||
{ 26, -0.2, { 1, 4, 2 }, 0.0 } };
|
||||
|
||||
// This is run with all possible oov setting and yields same result.
|
||||
void ReadSymbolicLmNoOovImpl(ArpaParseOptions::OovHandling oov) {
|
||||
int32 expect_counts[] = { 4, 2, 2 };
|
||||
TestSymbolTable symbols;
|
||||
symbols.AddSymbol("\xCE\xB2", 5);
|
||||
|
||||
ArpaParseOptions options;
|
||||
options.bos_symbol = 1;
|
||||
options.eos_symbol = 2;
|
||||
options.unk_symbol = 3;
|
||||
options.oov_handling = oov;
|
||||
TestableArpaFileParser parser(options, &symbols);
|
||||
std::istringstream stm(symbolic_lm, std::ios_base::in);
|
||||
parser.Read(stm);
|
||||
parser.Validate(MakeCountedArray(expect_counts),
|
||||
MakeCountedArray(expect_symbolic_full));
|
||||
KALDI_ASSERT(symbols.NumSymbols() == 6);
|
||||
}
|
||||
|
||||
void ReadSymbolicLmNoOovTests() {
|
||||
KALDI_LOG << "ReadSymbolicLmNoOovImpl(kRaiseError)";
|
||||
ReadSymbolicLmNoOovImpl(ArpaParseOptions::kRaiseError);
|
||||
KALDI_LOG << "ReadSymbolicLmNoOovImpl(kAddToSymbols)";
|
||||
ReadSymbolicLmNoOovImpl(ArpaParseOptions::kAddToSymbols);
|
||||
KALDI_LOG << "ReadSymbolicLmNoOovImpl(kReplaceWithUnk)";
|
||||
ReadSymbolicLmNoOovImpl(ArpaParseOptions::kReplaceWithUnk);
|
||||
KALDI_LOG << "ReadSymbolicLmNoOovImpl(kSkipNGram)";
|
||||
ReadSymbolicLmNoOovImpl(ArpaParseOptions::kSkipNGram);
|
||||
}
|
||||
|
||||
// This is run with all possible oov setting and yields same result.
|
||||
void ReadSymbolicLmWithOovImpl(
|
||||
ArpaParseOptions::OovHandling oov,
|
||||
CountedArray<NGramTestData> expect_ngrams,
|
||||
fst::SymbolTable* symbols) {
|
||||
int32 expect_counts[] = { 4, 2, 2 };
|
||||
ArpaParseOptions options;
|
||||
options.bos_symbol = 1;
|
||||
options.eos_symbol = 2;
|
||||
options.unk_symbol = 3;
|
||||
options.oov_handling = oov;
|
||||
TestableArpaFileParser parser(options, symbols);
|
||||
std::istringstream stm(symbolic_lm, std::ios_base::in);
|
||||
parser.Read(stm);
|
||||
parser.Validate(MakeCountedArray(expect_counts), expect_ngrams);
|
||||
}
|
||||
|
||||
void ReadSymbolicLmWithOovAddToSymbols() {
|
||||
TestSymbolTable symbols;
|
||||
ReadSymbolicLmWithOovImpl(ArpaParseOptions::kAddToSymbols,
|
||||
MakeCountedArray(expect_symbolic_full),
|
||||
&symbols);
|
||||
KALDI_ASSERT(symbols.NumSymbols() == 6);
|
||||
KALDI_ASSERT(symbols.Find("\xCE\xB2") == 5);
|
||||
}
|
||||
|
||||
void ReadSymbolicLmWithOovReplaceWithUnk() {
|
||||
NGramTestData expect_symbolic_unk_b[] = {
|
||||
{ 15, -5.2, { 4, 0, 0 }, -3.3 },
|
||||
{ 16, -3.4, { 3, 0, 0 }, 0.0 },
|
||||
{ 17, 0.0, { 1, 0, 0 }, -2.5 },
|
||||
{ 18, -4.3, { 2, 0, 0 }, 0.0 },
|
||||
|
||||
{ 21, -1.5, { 4, 3, 0 }, -3.2 },
|
||||
{ 22, -1.3, { 1, 4, 0 }, -4.2 },
|
||||
|
||||
{ 25, -0.3, { 1, 4, 3 }, 0.0 },
|
||||
{ 26, -0.2, { 1, 4, 2 }, 0.0 } };
|
||||
|
||||
TestSymbolTable symbols;
|
||||
ReadSymbolicLmWithOovImpl(ArpaParseOptions::kReplaceWithUnk,
|
||||
MakeCountedArray(expect_symbolic_unk_b),
|
||||
&symbols);
|
||||
KALDI_ASSERT(symbols.NumSymbols() == 5);
|
||||
}
|
||||
|
||||
void ReadSymbolicLmWithOovSkipNGram() {
|
||||
NGramTestData expect_symbolic_no_b[] = {
|
||||
{ 15, -5.2, { 4, 0, 0 }, -3.3 },
|
||||
{ 17, 0.0, { 1, 0, 0 }, -2.5 },
|
||||
{ 18, -4.3, { 2, 0, 0 }, 0.0 },
|
||||
|
||||
{ 22, -1.3, { 1, 4, 0 }, -4.2 },
|
||||
|
||||
{ 26, -0.2, { 1, 4, 2 }, 0.0 } };
|
||||
|
||||
TestSymbolTable symbols;
|
||||
ReadSymbolicLmWithOovImpl(ArpaParseOptions::kSkipNGram,
|
||||
MakeCountedArray(expect_symbolic_no_b),
|
||||
&symbols);
|
||||
KALDI_ASSERT(symbols.NumSymbols() == 5);
|
||||
}
|
||||
|
||||
void ReadSymbolicLmWithOovTests() {
|
||||
KALDI_LOG << "ReadSymbolicLmWithOovAddToSymbols()";
|
||||
ReadSymbolicLmWithOovAddToSymbols();
|
||||
KALDI_LOG << "ReadSymbolicLmWithOovReplaceWithUnk()";
|
||||
ReadSymbolicLmWithOovReplaceWithUnk();
|
||||
KALDI_LOG << "ReadSymbolicLmWithOovSkipNGram()";
|
||||
ReadSymbolicLmWithOovSkipNGram();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace kaldi
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
kaldi::ReadIntegerLmLogconvExpectSuccess();
|
||||
kaldi::ReadSymbolicLmNoOovTests();
|
||||
kaldi::ReadSymbolicLmWithOovTests();
|
||||
}
|
||||
Reference in New Issue
Block a user