Initial commit: FunASR Speech Recognition Toolkit
Update API Documentation / build-api-docs (push) Has been cancelled
Update API Documentation / build-api-docs (push) Has been cancelled
Add complete FunASR codebase including models, runtime, and documentation.
This commit is contained in:
freedakgmail
@@ -0,0 +1,30 @@
|
||||
|
||||
# make "all" the target.
|
||||
all:
|
||||
|
||||
# Disable linking math libs because not needed here. Just for compilation speed.
|
||||
# no, it's now needed for context-fst-test.
|
||||
# MATHLIB = NONE
|
||||
|
||||
EXTRA_CXXFLAGS = -Wno-sign-compare
|
||||
|
||||
include ../kaldi.mk
|
||||
|
||||
TESTFILES = determinize-star-test \
|
||||
pre-determinize-test trivial-factor-weight-test \
|
||||
context-fst-test factor-test table-matcher-test fstext-utils-test \
|
||||
remove-eps-local-test lattice-weight-test \
|
||||
determinize-lattice-test lattice-utils-test deterministic-fst-test \
|
||||
push-special-test epsilon-property-test prune-special-test
|
||||
|
||||
OBJFILES = push-special.o kaldi-fst-io.o context-fst.o grammar-context-fst.o
|
||||
|
||||
|
||||
LIBNAME = kaldi-fstext
|
||||
|
||||
# tree and matrix archives needed for test-context-fst
|
||||
# matrix archive needed for push-special.
|
||||
ADDLIBS = ../tree/kaldi-tree.a ../util/kaldi-util.a ../matrix/kaldi-matrix.a \
|
||||
../base/kaldi-base.a
|
||||
|
||||
include ../makefiles/default_rules.mk
|
||||
@@ -0,0 +1,256 @@
|
||||
// fstext/context-fst-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fstext/context-fst.h"
|
||||
#include "fstext/fst-test-utils.h"
|
||||
#include "tree/context-dep.h"
|
||||
#include "util/kaldi-io.h"
|
||||
#include "base/kaldi-math.h"
|
||||
|
||||
namespace fst
|
||||
{
|
||||
using std::vector;
|
||||
using std::cout;
|
||||
|
||||
// GenAcceptorFromSequence generates a linear acceptor (identical input+output symbols) that has this
|
||||
// sequence of symbols, and
|
||||
template<class Arc>
|
||||
static VectorFst<Arc> *GenAcceptorFromSequence(const vector<typename Arc::Label> &symbols, float cost) {
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef typename Arc::StateId StateId;
|
||||
|
||||
vector<float> split_cost(symbols.size()+1, 0.0); // for #-arcs + end-state.
|
||||
{ // compute split_cost. it must sum to "cost".
|
||||
std::set<int32> indices;
|
||||
size_t num_indices = 1 + (kaldi::Rand() % split_cost.size());
|
||||
while (indices.size() < num_indices) indices.insert(kaldi::Rand() % split_cost.size());
|
||||
for (std::set<int32>::iterator iter = indices.begin(); iter != indices.end(); ++iter) {
|
||||
split_cost[*iter] = cost / num_indices;
|
||||
}
|
||||
}
|
||||
|
||||
VectorFst<Arc> *fst = new VectorFst<Arc>();
|
||||
StateId cur_state = fst->AddState();
|
||||
fst->SetStart(cur_state);
|
||||
for (size_t i = 0; i < symbols.size(); i++) {
|
||||
StateId next_state = fst->AddState();
|
||||
Arc arc;
|
||||
arc.ilabel = symbols[i];
|
||||
arc.olabel = symbols[i];
|
||||
arc.nextstate = next_state;
|
||||
arc.weight = (Weight) split_cost[i];
|
||||
fst->AddArc(cur_state, arc);
|
||||
cur_state = next_state;
|
||||
|
||||
}
|
||||
fst->SetFinal(cur_state, (Weight)split_cost[symbols.size()]);
|
||||
return fst;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// CheckPhones is used to test the correctness of an FST that is the result of
|
||||
// composition with a ContextFst.
|
||||
template<class Arc>
|
||||
static float CheckPhones(const VectorFst<Arc> &linear_fst,
|
||||
const vector<typename Arc::Label> &phone_ids,
|
||||
const vector<typename Arc::Label> &disambig_ids,
|
||||
const vector<typename Arc::Label> &phone_seq,
|
||||
const vector<vector<typename Arc::Label> > &ilabel_info,
|
||||
int N, int P) {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
assert(kaldi::IsSorted(phone_ids)); // so we can do binary_search.
|
||||
|
||||
|
||||
vector<int32> input_syms;
|
||||
vector<int32> output_syms;
|
||||
Weight tot_cost;
|
||||
bool ans = GetLinearSymbolSequence(linear_fst, &input_syms,
|
||||
&output_syms, &tot_cost);
|
||||
assert(ans); // should be linear.
|
||||
|
||||
vector<int32> phone_seq_check;
|
||||
for (size_t i = 0; i < output_syms.size(); i++)
|
||||
if (std::binary_search(phone_ids.begin(), phone_ids.end(), output_syms[i]))
|
||||
phone_seq_check.push_back(output_syms[i]);
|
||||
|
||||
assert(phone_seq_check == phone_seq);
|
||||
|
||||
vector<vector<int32> > input_syms_long;
|
||||
for (size_t i = 0; i < input_syms.size(); i++) {
|
||||
Label isym = input_syms[i];
|
||||
if (ilabel_info[isym].size() == 0) continue; // epsilon.
|
||||
if ( (ilabel_info[isym].size() == 1 &&
|
||||
ilabel_info[isym][0] <= 0) ) continue; // disambig.
|
||||
input_syms_long.push_back(ilabel_info[isym]);
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < input_syms_long.size(); i++) {
|
||||
vector<int32> phone_context_window(N); // phone at pos i will be at pos P in this window.
|
||||
int pos = ((int)i) - P; // pos of first phone in window [ may be out of range] .
|
||||
for (int j = 0; j < N; j++, pos++) {
|
||||
if (static_cast<size_t>(pos) < phone_seq.size()) phone_context_window[j] = phone_seq[pos];
|
||||
else phone_context_window[j] = 0; // 0 is a special symbol that context-dep-itf expects to see
|
||||
// when no phone is present due to out-of-window. context-fst knows about this too.
|
||||
}
|
||||
assert(input_syms_long[i] == phone_context_window);
|
||||
}
|
||||
return tot_cost.Value();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
template<class Arc>
|
||||
static VectorFst<Arc> *GenRandPhoneSeq(vector<typename Arc::Label> &phone_syms,
|
||||
vector<typename Arc::Label> &disambig_syms,
|
||||
typename Arc::Label subsequential_symbol,
|
||||
int num_subseq_syms,
|
||||
float seq_prob,
|
||||
vector<typename Arc::Label> *phoneseq_out) {
|
||||
KALDI_ASSERT(phoneseq_out != NULL);
|
||||
typedef typename Arc::Label Label;
|
||||
// Generate an FST that is a random phone sequence, ending
|
||||
// with "num_subseq_syms" subsequential symbols. It will
|
||||
// have disambiguation symbols randomly interspersed throughout.
|
||||
// The number of phones is random (possibly zero).
|
||||
size_t len = (kaldi::Rand() % 4) * (kaldi::Rand() % 3); // up to 3*2=6 phones.
|
||||
float disambig_prob = 0.33;
|
||||
phoneseq_out->clear();
|
||||
vector<Label> syms; // the phones
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
while (kaldi::RandUniform() < disambig_prob) syms.push_back(disambig_syms[kaldi::Rand() % disambig_syms.size()]);
|
||||
Label phone_id = phone_syms[kaldi::Rand() % phone_syms.size()];
|
||||
phoneseq_out->push_back(phone_id); // record in output the underlying phone sequence.
|
||||
syms.push_back(phone_id);
|
||||
}
|
||||
for (size_t i = 0; static_cast<int32>(i) < num_subseq_syms; i++) {
|
||||
while (kaldi::RandUniform() < disambig_prob) syms.push_back(disambig_syms[kaldi::Rand() % disambig_syms.size()]);
|
||||
syms.push_back(subsequential_symbol);
|
||||
}
|
||||
while (kaldi::RandUniform() < disambig_prob) syms.push_back(disambig_syms[kaldi::Rand() % disambig_syms.size()]);
|
||||
|
||||
// OK, now have the symbols of the FST as a vector.
|
||||
return GenAcceptorFromSequence<Arc>(syms, seq_prob);
|
||||
}
|
||||
|
||||
// Don't instantiate with log semiring, as RandEquivalent may fail.
|
||||
// TestContestFst also test ReadILabelInfo and WriteILabelInfo.
|
||||
static void TestContextFst(bool verbose, bool use_matcher) {
|
||||
typedef StdArc Arc;
|
||||
typedef Arc::Label Label;
|
||||
typedef Arc::StateId StateId;
|
||||
typedef Arc::Weight Weight;
|
||||
|
||||
// Generate a random set of phones.
|
||||
size_t num_phones = 1 + kaldi::Rand() % 10;
|
||||
std::set<int32> phones_set;
|
||||
while (phones_set.size() < num_phones) phones_set.insert(1 + kaldi::Rand() % (num_phones + 5)); // don't use 0 [== epsilon]
|
||||
vector<int32> phones;
|
||||
kaldi::CopySetToVector(phones_set, &phones);
|
||||
|
||||
int N = 1 + kaldi::Rand() % 4; // Context size, in range 1..4.
|
||||
int P = kaldi::Rand() % N; // 1.. N-1.
|
||||
if (verbose) std::cout << "N = "<< N << ", P = "<<P<<'\n';
|
||||
|
||||
Label subsequential_symbol = 1000;
|
||||
vector<int32> disambig_syms;
|
||||
for (size_t i =0; i < 5; i++) disambig_syms.push_back(500 + i);
|
||||
vector<int32> phone_syms;
|
||||
for (size_t i = 0; i < phones.size();i++) phone_syms.push_back(phones[i]);
|
||||
|
||||
|
||||
InverseContextFst inv_cfst(subsequential_symbol,
|
||||
phones, disambig_syms,
|
||||
N, P);
|
||||
|
||||
|
||||
/* Now create random phone-sequences and compose them with the context FST.
|
||||
*/
|
||||
|
||||
for (size_t p = 0; p < 10; p++) {
|
||||
vector<int32> phone_seq;
|
||||
int num_subseq = N - P - 1; // zero if P == N-1, i.e. P is last element, i.e. left-context only.
|
||||
float tot_cost = 20.0 * kaldi::RandUniform();
|
||||
VectorFst<Arc> *f = GenRandPhoneSeq<Arc>(phone_syms, disambig_syms, subsequential_symbol, num_subseq, tot_cost, &phone_seq);
|
||||
if (verbose) {
|
||||
std::cout << "Sequence FST is:\n";
|
||||
{ // Try to print the fst.
|
||||
FstPrinter<Arc> fstprinter(*f, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
}
|
||||
|
||||
VectorFst<Arc> fst_composed;
|
||||
|
||||
ComposeDeterministicOnDemandInverse(*f, &inv_cfst, &fst_composed);
|
||||
|
||||
|
||||
// Testing WriteILabelInfo and ReadILabelInfo.
|
||||
{
|
||||
bool binary = (kaldi::Rand() % 2 == 0);
|
||||
WriteILabelInfo(kaldi::Output("tmpf", binary).Stream(),
|
||||
binary, inv_cfst.IlabelInfo());
|
||||
|
||||
bool binary_in;
|
||||
vector<vector<int32> > ilabel_info;
|
||||
kaldi::Input ki("tmpf", &binary_in);
|
||||
ReadILabelInfo(ki.Stream(),
|
||||
binary_in, &ilabel_info);
|
||||
assert(ilabel_info == inv_cfst.IlabelInfo());
|
||||
}
|
||||
|
||||
|
||||
if (verbose) {
|
||||
std::cout << "Composed FST is:\n";
|
||||
{ // Try to print the fst.
|
||||
FstPrinter<Arc> fstprinter(fst_composed, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
}
|
||||
|
||||
// now check the composed FST.
|
||||
float tot_cost_check = CheckPhones<Arc>(fst_composed,
|
||||
phone_syms,
|
||||
disambig_syms,
|
||||
phone_seq,
|
||||
inv_cfst.IlabelInfo(),
|
||||
N, P);
|
||||
kaldi::AssertEqual(tot_cost, tot_cost_check);
|
||||
|
||||
delete f;
|
||||
}
|
||||
|
||||
unlink("tmpf");
|
||||
}
|
||||
|
||||
|
||||
} // namespace fst
|
||||
|
||||
int main() {
|
||||
|
||||
for (int i = 0;i < 16;i++) {
|
||||
bool verbose = (i < 4);
|
||||
bool use_matcher = ( (i/4) % 2 == 0);
|
||||
fst::TestContextFst(verbose, use_matcher);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
// fstext/context-fst.cc
|
||||
|
||||
// Copyright 2018 Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fstext/context-fst.h"
|
||||
#include "base/kaldi-error.h"
|
||||
|
||||
namespace fst {
|
||||
using std::vector;
|
||||
|
||||
|
||||
InverseContextFst::InverseContextFst(
|
||||
Label subsequential_symbol,
|
||||
const vector<int32>& phones,
|
||||
const vector<int32>& disambig_syms,
|
||||
int32 context_width,
|
||||
int32 central_position):
|
||||
context_width_(context_width),
|
||||
central_position_(central_position),
|
||||
phone_syms_(phones),
|
||||
disambig_syms_(disambig_syms),
|
||||
subsequential_symbol_(subsequential_symbol) {
|
||||
|
||||
{ // This block checks the inputs.
|
||||
KALDI_ASSERT(subsequential_symbol != 0
|
||||
&& disambig_syms_.count(subsequential_symbol) == 0
|
||||
&& phone_syms_.count(subsequential_symbol) == 0);
|
||||
if (phone_syms_.empty())
|
||||
KALDI_WARN << "Context FST created but there are no phone symbols: probably "
|
||||
"input FST was empty.";
|
||||
KALDI_ASSERT(phone_syms_.count(0) == 0 && disambig_syms_.count(0) == 0 &&
|
||||
central_position_ >= 0 && central_position_ < context_width_);
|
||||
for (size_t i = 0; i < phones.size(); i++) {
|
||||
KALDI_ASSERT(disambig_syms_.count(phones[i]) == 0);
|
||||
}
|
||||
}
|
||||
|
||||
// empty vector, will be the ilabel_info vector that corresponds to epsilon,
|
||||
// in case our FST needs to output epsilons.
|
||||
vector<int32> empty_vec;
|
||||
Label epsilon_label = FindLabel(empty_vec);
|
||||
|
||||
// epsilon_vec is the phonetic context window we have at the very start of a
|
||||
// sequence, meaning "no real phones have been seen yet".
|
||||
vector<int32> epsilon_vec(context_width_ - 1, 0);
|
||||
StateId start_state = FindState(epsilon_vec);
|
||||
|
||||
KALDI_ASSERT(epsilon_label == 0 && start_state == 0);
|
||||
|
||||
if (context_width_ > central_position_ + 1 && !disambig_syms_.empty()) {
|
||||
// We add a symbol whose sequence representation is [ 0 ], and whose
|
||||
// symbol-id is 1. This is treated as a disambiguation symbol, we call it
|
||||
// #-1 in printed form. It is necessary to ensure that all determinizable
|
||||
// LG's will have determinizable CLG's. The problem it fixes is quite
|
||||
// subtle-- it relates to reordering of disambiguation symbols (they appear
|
||||
// earlier in CLG than in LG, relative to phones), and the fact that if a
|
||||
// disambig symbol appears at the very start of a sequence in CLG, it's not
|
||||
// clear exatly where it appeared on the corresponding sequence at the input
|
||||
// of LG.
|
||||
vector<int32> pseudo_eps_vec;
|
||||
pseudo_eps_vec.push_back(0);
|
||||
pseudo_eps_symbol_= FindLabel(pseudo_eps_vec);
|
||||
KALDI_ASSERT(pseudo_eps_symbol_ == 1);
|
||||
} else {
|
||||
pseudo_eps_symbol_ = 0; // use actual epsilon.
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void InverseContextFst::ShiftSequenceLeft(Label label,
|
||||
std::vector<int32> *phone_seq) {
|
||||
if (!phone_seq->empty()) {
|
||||
phone_seq->erase(phone_seq->begin());
|
||||
phone_seq->push_back(label);
|
||||
}
|
||||
}
|
||||
|
||||
void InverseContextFst::GetFullPhoneSequence(
|
||||
const std::vector<int32> &seq, Label label,
|
||||
std::vector<int32> *full_phone_sequence) {
|
||||
int32 context_width = context_width_;
|
||||
full_phone_sequence->reserve(context_width);
|
||||
full_phone_sequence->insert(full_phone_sequence->end(),
|
||||
seq.begin(), seq.end());
|
||||
full_phone_sequence->push_back(label);
|
||||
for (int32 i = central_position_ + 1; i < context_width; i++) {
|
||||
if ((*full_phone_sequence)[i] == subsequential_symbol_) {
|
||||
(*full_phone_sequence)[i] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
InverseContextFst::Weight InverseContextFst::Final(StateId s) {
|
||||
KALDI_ASSERT(static_cast<size_t>(s) < state_seqs_.size());
|
||||
|
||||
const vector<int32> &phone_context = state_seqs_[s];
|
||||
|
||||
KALDI_ASSERT(phone_context.size() == context_width_ - 1);
|
||||
|
||||
bool has_final_prob;
|
||||
|
||||
if (central_position_ < context_width_ - 1) {
|
||||
has_final_prob = (phone_context[central_position_] == subsequential_symbol_);
|
||||
// if phone_context[central_position_] != subsequential_symbol_ then we have
|
||||
// pending phones-in-context that we still need to output, so we need to
|
||||
// consume more subsequential symbols before we can terminate.
|
||||
} else {
|
||||
has_final_prob = true;
|
||||
}
|
||||
return has_final_prob ? Weight::One() : Weight::Zero();
|
||||
}
|
||||
|
||||
bool InverseContextFst::GetArc(StateId s, Label ilabel, Arc *arc) {
|
||||
KALDI_ASSERT(ilabel != 0 && static_cast<size_t>(s) < state_seqs_.size() &&
|
||||
state_seqs_[s].size() == context_width_ - 1);
|
||||
|
||||
if (IsDisambigSymbol(ilabel)) {
|
||||
// A disambiguation-symbol self-loop arc.
|
||||
CreateDisambigArc(s, ilabel, arc);
|
||||
return true;
|
||||
} else if (IsPhoneSymbol(ilabel)) {
|
||||
const vector<int32> &seq = state_seqs_[s];
|
||||
if (!seq.empty() && seq.back() == subsequential_symbol_) {
|
||||
return false; // A real phone is not allowed to follow the subsequential
|
||||
// symbol.
|
||||
}
|
||||
|
||||
// next_seq will be 'seq' shifted left by 1, with 'ilabel' appended.
|
||||
vector<int32> next_seq(seq);
|
||||
ShiftSequenceLeft(ilabel, &next_seq);
|
||||
|
||||
// full-seq will be the full context window of size context_width_.
|
||||
vector<int32> full_seq;
|
||||
GetFullPhoneSequence(seq, ilabel, &full_seq);
|
||||
|
||||
StateId next_s = FindState(next_seq);
|
||||
|
||||
CreatePhoneOrEpsArc(s, next_s, ilabel, full_seq, arc);
|
||||
return true;
|
||||
} else if (ilabel == subsequential_symbol_) {
|
||||
const vector<int32> &seq = state_seqs_[s];
|
||||
|
||||
if (central_position_ + 1 == context_width_ ||
|
||||
seq[central_position_] == subsequential_symbol_) {
|
||||
// We already had "enough" subsequential symbols in a row and don't want to
|
||||
// accept any more, or we'd be making the subsequential symbol the central phone.
|
||||
return false;
|
||||
}
|
||||
|
||||
// full-seq will be the full context window of size context_width_.
|
||||
vector<int32> full_seq;
|
||||
GetFullPhoneSequence(seq, ilabel, &full_seq);
|
||||
|
||||
vector<int32> next_seq(seq);
|
||||
ShiftSequenceLeft(ilabel, &next_seq);
|
||||
StateId next_s = FindState(next_seq);
|
||||
|
||||
CreatePhoneOrEpsArc(s, next_s, ilabel, full_seq, arc);
|
||||
return true;
|
||||
} else {
|
||||
KALDI_ERR << "ContextFst: CreateArc, invalid ilabel supplied [confusion "
|
||||
<< "about phone list or disambig symbols?]: " << ilabel;
|
||||
}
|
||||
return false; // won't get here. suppress compiler error.
|
||||
}
|
||||
|
||||
|
||||
void InverseContextFst::CreateDisambigArc(StateId s, Label ilabel, Arc *arc) {
|
||||
// Creates a self-loop arc corresponding to the disambiguation symbol.
|
||||
vector<int32> label_info; // This will be a vector containing just [ -olabel ].
|
||||
label_info.push_back(-ilabel); // olabel is a disambiguation symbol. Use its negative
|
||||
// so we can more easily distinguish them from phones.
|
||||
Label olabel = FindLabel(label_info);
|
||||
arc->ilabel = ilabel;
|
||||
arc->olabel = olabel;
|
||||
arc->weight = Weight::One();
|
||||
arc->nextstate = s; // self-loop.
|
||||
}
|
||||
|
||||
void InverseContextFst::CreatePhoneOrEpsArc(StateId src, StateId dest,
|
||||
Label ilabel,
|
||||
const vector<int32> &phone_seq,
|
||||
Arc *arc) {
|
||||
KALDI_PARANOID_ASSERT(phone_seq[central_position_] != subsequential_symbol_);
|
||||
|
||||
arc->ilabel = ilabel;
|
||||
arc->weight = Weight::One();
|
||||
arc->nextstate = dest;
|
||||
if (phone_seq[central_position_] == 0) {
|
||||
// This can happen at the beginning of the graph. In this case we don't
|
||||
// output a real phone, we createdt an epsilon arc (but sometimes we need to
|
||||
// use a special disambiguation symbol instead of epsilon).
|
||||
arc->olabel = pseudo_eps_symbol_;
|
||||
} else {
|
||||
// We have a phone in the central position.
|
||||
arc->olabel = FindLabel(phone_seq);
|
||||
}
|
||||
}
|
||||
|
||||
StdArc::StateId InverseContextFst::FindState(const vector<int32> &seq) {
|
||||
// Finds state-id corresponding to this vector of phones. Inserts it if
|
||||
// necessary.
|
||||
KALDI_ASSERT(static_cast<int32>(seq.size()) == context_width_ - 1);
|
||||
VectorToStateMap::const_iterator iter = state_map_.find(seq);
|
||||
if (iter == state_map_.end()) { // Not already in map.
|
||||
StateId this_state_id = (StateId)state_seqs_.size();
|
||||
state_seqs_.push_back(seq);
|
||||
state_map_[seq] = this_state_id;
|
||||
return this_state_id;
|
||||
} else {
|
||||
return iter->second;
|
||||
}
|
||||
}
|
||||
|
||||
StdArc::Label InverseContextFst::FindLabel(const vector<int32> &label_vec) {
|
||||
// Finds the ilabel corresponding to this vector (creates a new ilabel if
|
||||
// necessary).
|
||||
VectorToLabelMap::const_iterator iter = ilabel_map_.find(label_vec);
|
||||
if (iter == ilabel_map_.end()) { // Not already in map.
|
||||
Label this_label = ilabel_info_.size();
|
||||
ilabel_info_.push_back(label_vec);
|
||||
ilabel_map_[label_vec] = this_label;
|
||||
return this_label;
|
||||
} else {
|
||||
return iter->second;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void ComposeContext(const vector<int32> &disambig_syms_in,
|
||||
int32 context_width, int32 central_position,
|
||||
VectorFst<StdArc> *ifst,
|
||||
VectorFst<StdArc> *ofst,
|
||||
vector<vector<int32> > *ilabels_out,
|
||||
bool project_ifst) {
|
||||
KALDI_ASSERT(ifst != NULL && ofst != NULL);
|
||||
KALDI_ASSERT(context_width > 0);
|
||||
KALDI_ASSERT(central_position >= 0);
|
||||
KALDI_ASSERT(central_position < context_width);
|
||||
|
||||
vector<int32> disambig_syms(disambig_syms_in);
|
||||
std::sort(disambig_syms.begin(), disambig_syms.end());
|
||||
|
||||
vector<int32> all_syms;
|
||||
GetInputSymbols(*ifst, false/*no eps*/, &all_syms);
|
||||
std::sort(all_syms.begin(), all_syms.end());
|
||||
vector<int32> phones;
|
||||
for (size_t i = 0; i < all_syms.size(); i++)
|
||||
if (!std::binary_search(disambig_syms.begin(),
|
||||
disambig_syms.end(), all_syms[i]))
|
||||
phones.push_back(all_syms[i]);
|
||||
|
||||
// Get subsequential symbol that does not clash with
|
||||
// any disambiguation symbol or symbol in the FST.
|
||||
int32 subseq_sym = 1;
|
||||
if (!all_syms.empty())
|
||||
subseq_sym = std::max(subseq_sym, all_syms.back() + 1);
|
||||
if (!disambig_syms.empty())
|
||||
subseq_sym = std::max(subseq_sym, disambig_syms.back() + 1);
|
||||
|
||||
// if central_position == context_width-1, it's left-context, and no
|
||||
// subsequential symbol is needed.
|
||||
if (central_position != context_width-1) {
|
||||
AddSubsequentialLoop(subseq_sym, ifst);
|
||||
if (project_ifst) {
|
||||
fst::Project(ifst, fst::PROJECT_INPUT);
|
||||
}
|
||||
}
|
||||
|
||||
InverseContextFst inv_c(subseq_sym, phones, disambig_syms,
|
||||
context_width, central_position);
|
||||
|
||||
// The following statement is equivalent to the following
|
||||
// (if FSTs had the '*' operator for composition):
|
||||
// (*ofst) = inv(inv_c) * (*ifst)
|
||||
ComposeDeterministicOnDemandInverse(*ifst, &inv_c, ofst);
|
||||
|
||||
inv_c.SwapIlabelInfo(ilabels_out);
|
||||
}
|
||||
|
||||
void AddSubsequentialLoop(StdArc::Label subseq_symbol,
|
||||
MutableFst<StdArc> *fst) {
|
||||
typedef StdArc Arc;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
vector<StateId> final_states;
|
||||
for (StateIterator<MutableFst<Arc> > siter(*fst); !siter.Done(); siter.Next()) {
|
||||
StateId s = siter.Value();
|
||||
if (fst->Final(s) != Weight::Zero()) final_states.push_back(s);
|
||||
}
|
||||
|
||||
StateId superfinal = fst->AddState();
|
||||
Arc arc(subseq_symbol, 0, Weight::One(), superfinal);
|
||||
fst->AddArc(superfinal, arc); // loop at superfinal.
|
||||
fst->SetFinal(superfinal, Weight::One());
|
||||
|
||||
for (size_t i = 0; i < final_states.size(); i++) {
|
||||
StateId s = final_states[i];
|
||||
fst->AddArc(s, Arc(subseq_symbol, 0, fst->Final(s), superfinal));
|
||||
// No, don't remove the final-weights of the original states..
|
||||
// this is so we can add the subsequential loop in cases where
|
||||
// there is no context, and it won't hurt.
|
||||
// fst->SetFinal(s, Weight::Zero());
|
||||
arc.nextstate = final_states[i];
|
||||
}
|
||||
}
|
||||
|
||||
void WriteILabelInfo(std::ostream &os, bool binary,
|
||||
const vector<vector<int32> > &info) {
|
||||
int32 size = info.size();
|
||||
kaldi::WriteBasicType(os, binary, size);
|
||||
for (int32 i = 0; i < size; i++) {
|
||||
kaldi::WriteIntegerVector(os, binary, info[i]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void ReadILabelInfo(std::istream &is, bool binary,
|
||||
vector<vector<int32> > *info) {
|
||||
int32 size = info->size();
|
||||
kaldi::ReadBasicType(is, binary, &size);
|
||||
info->resize(size);
|
||||
for (int32 i = 0; i < size; i++) {
|
||||
kaldi::ReadIntegerVector(is, binary, &((*info)[i]));
|
||||
}
|
||||
}
|
||||
|
||||
SymbolTable *CreateILabelInfoSymbolTable(const vector<vector<int32> > &info,
|
||||
const SymbolTable &phones_symtab,
|
||||
std::string separator,
|
||||
std::string initial_disambig) { // e.g. separator = "/", initial-disambig="#-1"
|
||||
KALDI_ASSERT(!info.empty() && info[0].empty());
|
||||
SymbolTable *ans = new SymbolTable("ilabel-info-symtab");
|
||||
int64 s = ans->AddSymbol(phones_symtab.Find(static_cast<int64>(0)));
|
||||
assert(s == 0);
|
||||
for (size_t i = 1; i < info.size(); i++) {
|
||||
if (info[i].size() == 0) {
|
||||
KALDI_ERR << "Invalid ilabel-info";
|
||||
}
|
||||
if (info[i].size() == 1 &&
|
||||
info[i][0] <= 0) {
|
||||
if (info[i][0] == 0) { // special symbol at start that we want to call #-1.
|
||||
s = ans->AddSymbol(initial_disambig);
|
||||
if (s != i) {
|
||||
KALDI_ERR << "Disambig symbol " << initial_disambig
|
||||
<< " already in vocab";
|
||||
}
|
||||
} else {
|
||||
std::string disambig_sym = phones_symtab.Find(-info[i][0]);
|
||||
if (disambig_sym == "") {
|
||||
KALDI_ERR << "Disambig symbol " << -info[i][0]
|
||||
<< " not in phone symbol-table";
|
||||
}
|
||||
s = ans->AddSymbol(disambig_sym);
|
||||
if (s != i) {
|
||||
KALDI_ERR << "Disambig symbol " << disambig_sym
|
||||
<< " already in vocab";
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// is a phone-context-window.
|
||||
std::string newsym;
|
||||
for (size_t j = 0; j < info[i].size(); j++) {
|
||||
std::string phonesym = phones_symtab.Find(info[i][j]);
|
||||
if (phonesym == "") {
|
||||
KALDI_ERR << "Symbol " << info[i][j]
|
||||
<< " not in phone symbol-table";
|
||||
}
|
||||
if (j != 0) newsym += separator;
|
||||
newsym += phonesym;
|
||||
}
|
||||
int64 s = ans->AddSymbol(newsym);
|
||||
if (s != static_cast<int64>(i)) {
|
||||
KALDI_ERR << "Some problem with duplicate symbols";
|
||||
}
|
||||
}
|
||||
}
|
||||
return ans;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
} // end namespace fst
|
||||
@@ -0,0 +1,340 @@
|
||||
// fstext/context-fst.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2018 Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// This file includes material from the OpenFST Library v1.2.7 available at
|
||||
// http://www.openfst.org and released under the Apache License Version 2.0.
|
||||
//
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Copyright 2005-2010 Google, Inc.
|
||||
// Author: riley@google.com (Michael Riley)
|
||||
|
||||
|
||||
|
||||
#ifndef KALDI_FSTEXT_CONTEXT_FST_H_
|
||||
#define KALDI_FSTEXT_CONTEXT_FST_H_
|
||||
|
||||
/* This header defines a context FST "C" (the "C" in "HCLG") which transduces
|
||||
from symbols representing phone context windows (e.g. "a, b, c") to
|
||||
individual phones, e.g. "a". Search for "hbka.pdf" ("Speech Recognition
|
||||
with Weighted Finite State Transducers") by M. Mohri, for more context.
|
||||
*/
|
||||
|
||||
#include <unordered_map>
|
||||
using std::unordered_map;
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
|
||||
#include "util/const-integer-set.h"
|
||||
#include "fstext/deterministic-fst.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
|
||||
|
||||
/// Utility function for writing ilabel-info vectors to disk.
|
||||
void WriteILabelInfo(std::ostream &os, bool binary,
|
||||
const std::vector<std::vector<int32> > &ilabel_info);
|
||||
|
||||
/// Utility function for reading ilabel-info vectors from disk.
|
||||
void ReadILabelInfo(std::istream &is, bool binary,
|
||||
std::vector<std::vector<int32> > *ilabel_info);
|
||||
|
||||
|
||||
/// The following function is mainly of use for printing and debugging.
|
||||
SymbolTable *CreateILabelInfoSymbolTable(const std::vector<std::vector<int32> > &ilabel_info,
|
||||
const SymbolTable &phones_symtab,
|
||||
std::string separator,
|
||||
std::string disambig_prefix); // e.g. separator = "/", disambig_prefix = "#"
|
||||
|
||||
|
||||
|
||||
/**
|
||||
Used in the command-line tool fstcomposecontext. It creates a context FST and
|
||||
composes it on the left with "ifst" to make "ofst". It outputs the label
|
||||
information to ilabels_out. "ifst" is mutable because we need to add the
|
||||
subsequential loop.
|
||||
|
||||
@param [in] disambig_syms List of disambiguation symbols, e.g. the integer
|
||||
ids of #0, #1, #2 ... in the phones.txt.
|
||||
@param [in] context_width Size of context window, e.g. 3 for triphone.
|
||||
@param [in] central_position Central position in phonetic context window
|
||||
(zero-based index), e.g. 1 for triphone.
|
||||
@param [in,out] ifst The FST we are composing with C (e.g. LG.fst), mustable because
|
||||
we need to add the subsequential loop to it.
|
||||
@param [out] ofst Composed output FST (would be CLG.fst).
|
||||
@param [out] ilabels_out Vector, indexed by ilabel of CLG.fst, providing information
|
||||
about the meaning of that ilabel; see
|
||||
"http://kaldi-asr.org/doc/tree_externals.html#tree_ilabel".
|
||||
@param [in] project_ifst This is intended only to be set to true
|
||||
in the program 'fstmakecontextfst'... if true, it will
|
||||
project on the input after adding the subsequential loop
|
||||
to 'ifst', which allows us to reconstruct the context
|
||||
fst C.fst.
|
||||
*/
|
||||
void ComposeContext(const std::vector<int32> &disambig_syms,
|
||||
int32 context_width, int32 central_position,
|
||||
VectorFst<StdArc> *ifst,
|
||||
VectorFst<StdArc> *ofst,
|
||||
std::vector<std::vector<int32> > *ilabels_out,
|
||||
bool project_ifst = false);
|
||||
|
||||
|
||||
/**
|
||||
Modifies an FST so that it transuces the same paths, but the input side of the
|
||||
paths can all have the subsequential symbol '$' appended to them any number of
|
||||
times (we could easily specify the number of times, but accepting any number of
|
||||
repetitions is just more convenient). The actual way we do this is for each
|
||||
final state, we add a transition with weight equal to the final-weight of that
|
||||
state, with input-symbol '$' and output-symbols \<eps\>, and ending in a new
|
||||
super-final state that has unit final-probability and a unit-weight self-loop
|
||||
with '$' on its input and \<eps\> on its output. The reason we don't just
|
||||
add a loop to each final-state has to do with preserving stochasticity
|
||||
(see \ref fst_algo_stochastic). We keep the final-probability in all the
|
||||
original final-states rather than setting them to zero, so the resulting FST
|
||||
can accept zero '$' symbols at the end (in case we had no right context).
|
||||
*/
|
||||
void AddSubsequentialLoop(StdArc::Label subseq_symbol,
|
||||
MutableFst<StdArc> *fst);
|
||||
|
||||
|
||||
/*
|
||||
InverseContextFst represents the inverse of the context FST "C" (the "C" in
|
||||
"HCLG") which transduces from symbols representing phone context windows
|
||||
(e.g. "a, b, c") to individual phones, e.g. "a". So InverseContextFst
|
||||
transduces from phones to symbols representing phone context windows. The
|
||||
point is that the inverse is deterministic, so the DeterministicOnDemandFst
|
||||
interface is applicable, which turns out to be a convenient way to implement
|
||||
this.
|
||||
|
||||
This doesn't implement the full Fst interface, it implements the
|
||||
DeterministicOnDemandFst interface which is much simpler and which is
|
||||
sufficient for what we need to do with this.
|
||||
|
||||
Search for "hbka.pdf" ("Speech Recognition with Weighted Finite State
|
||||
Transducers") by M. Mohri, for more context.
|
||||
*/
|
||||
|
||||
class InverseContextFst: public DeterministicOnDemandFst<StdArc> {
|
||||
public:
|
||||
typedef StdArc Arc;
|
||||
typedef typename StdArc::StateId StateId;
|
||||
typedef typename StdArc::Weight Weight;
|
||||
typedef typename StdArc::Label Label;
|
||||
|
||||
/**
|
||||
Constructor.
|
||||
@param [in] subsequential_symbol The integer id of the 'subsequential symbol'
|
||||
(usually represented as '$') that terminates sequences on the
|
||||
output of C.fst (input of InverseContextFst). Search for
|
||||
"quential" in https://cs.nyu.edu/~mohri/pub/hbka.pdf.
|
||||
This may just be the first unused integer id. Must be nonzer.
|
||||
@param [in] phones List of integer ids of phones, as you would see in phones.txt
|
||||
@param [in] disambig_syms List of integer ids of disambiguation symbols,
|
||||
e.g. the ids of #0, #1, #2 in phones.txt
|
||||
@param [in] context_width Size of context window, e.g. 3 for triphone.
|
||||
@param [in] central_position Central position in context window (zero-based),
|
||||
e.g. 1 for triphone.
|
||||
See \ref graph_context for more details.
|
||||
*/
|
||||
InverseContextFst(Label subsequential_symbol,
|
||||
const std::vector<int32>& phones,
|
||||
const std::vector<int32>& disambig_syms,
|
||||
int32 context_width,
|
||||
int32 central_position);
|
||||
|
||||
|
||||
virtual StateId Start() { return 0; }
|
||||
|
||||
virtual Weight Final(StateId s);
|
||||
|
||||
/// Note: ilabel must not be epsilon.
|
||||
virtual bool GetArc(StateId s, Label ilabel, Arc *arc);
|
||||
|
||||
~InverseContextFst() { }
|
||||
|
||||
// Returns a reference to a vector<vector<int32> > with information about all
|
||||
// the input symbols of C (i.e. all the output symbols of this
|
||||
// InverseContextFst). See
|
||||
// "http://kaldi-asr.org/doc/tree_externals.html#tree_ilabel".
|
||||
const std::vector<std::vector<int32> > &IlabelInfo() const {
|
||||
return ilabel_info_;
|
||||
}
|
||||
|
||||
// A way to destructively obtain the ilabel-info. Only do this if you
|
||||
// are just about to destroy this object.
|
||||
void SwapIlabelInfo(std::vector<std::vector<int32> > *vec) { ilabel_info_.swap(*vec); }
|
||||
|
||||
private:
|
||||
|
||||
/// Returns the state-id corresponding to this vector of phones; creates the
|
||||
/// state it if necessary. Requires seq.size() == context_width_ - 1.
|
||||
StateId FindState(const std::vector<int32> &seq);
|
||||
|
||||
/// Finds the label index corresponding to this context-window of phones
|
||||
/// (likely of width context_width_). Inserts it into the
|
||||
/// ilabel_info_/ilabel_map_ tables if necessary.
|
||||
Label FindLabel(const std::vector<int32> &label_info);
|
||||
|
||||
inline bool IsDisambigSymbol(Label lab) { return (disambig_syms_.count(lab) != 0); }
|
||||
|
||||
inline bool IsPhoneSymbol(Label lab) { return (phone_syms_.count(lab) != 0); }
|
||||
|
||||
/// Create disambiguation-symbol self-loop arc; where 'ilabel' must correspond to
|
||||
/// a disambiguation symbol. Called from CreateArc().
|
||||
inline void CreateDisambigArc(StateId s, Label ilabel, Arc *arc);
|
||||
|
||||
/// Creates an arc, this function is to be called only when 'ilabel'
|
||||
/// corresponds to a phone. Called from CreateArc(). The olabel may end be
|
||||
/// epsilon, instead of a phone-in-context, if the system has right context
|
||||
/// and we are very near the beginning of the phone sequence.
|
||||
inline void CreatePhoneOrEpsArc(StateId src, StateId dst, Label ilabel,
|
||||
const std::vector<int32> &phone_seq, Arc *arc);
|
||||
|
||||
|
||||
/// If phone_seq is nonempty then this function it left by one and appends
|
||||
/// 'label' to it, otherwise it does nothing. We expect (but do not check)
|
||||
/// that phone_seq->size() == context_width_ - 1.
|
||||
inline void ShiftSequenceLeft(Label label, std::vector<int32> *phone_seq);
|
||||
|
||||
/// This utility function does something equivalent to the following 3 steps:
|
||||
/// *full_phone_sequence = seq;
|
||||
/// full_phone_sequence->append(label)
|
||||
/// Replace any values equal to 'subsequential_symbol_' in
|
||||
/// full_phone_sequence with zero (this is to avoid having to keep track of
|
||||
/// the value of 'subsequential_symbol_' outside of this program).
|
||||
/// This function assumes that seq.size() == context_width_ - 1, and also that
|
||||
/// 'subsequential_symbol_' does not appear in positions 0 through
|
||||
/// central_position_ of 'seq'.
|
||||
inline void GetFullPhoneSequence(const std::vector<int32> &seq, Label label,
|
||||
std::vector<int32> *full_phone_sequence);
|
||||
|
||||
// Map type to map from vectors of int32 (representing phonetic contexts,
|
||||
// which will be of dimension context_width - 1) to StateId (corresponding to
|
||||
// the state index in this FST).
|
||||
typedef unordered_map<std::vector<int32>, StateId,
|
||||
kaldi::VectorHasher<int32> > VectorToStateMap;
|
||||
|
||||
// Map type to map from vectors of int32 (representing ilabel-info,
|
||||
// see http://kaldi-asr.org/doc/tree_externals.html#tree_ilabel) to
|
||||
// Label (the output label in this FST).
|
||||
typedef unordered_map<std::vector<int32>, Label,
|
||||
kaldi::VectorHasher<int32> > VectorToLabelMap;
|
||||
|
||||
|
||||
// Sometimes called N, context_width_ this is the width of the
|
||||
// phonetic context, e.g. 3 for triphone, 2 for biphone, one for monophone.
|
||||
// It is a user-specified value.
|
||||
int32 context_width_;
|
||||
|
||||
// Sometimes called P, central_position_ is is the (zero-based) "central
|
||||
// position" in the context window, meaning the phone that is "in" a certain
|
||||
// context. The most widely used values of (context-width, central-position)
|
||||
// are: (3,1) for triphone, (1,0) for monophone, and (2, 1) for left biphone.
|
||||
// This is also specified by the user. As an example, in the left-biphone
|
||||
// [ 5, 6 ], we view it as "the phone numbered 6 with the phone numbered 5 as
|
||||
// its left-context".
|
||||
int32 central_position_;
|
||||
|
||||
// The following three variables were also passed in by the caller:
|
||||
|
||||
// 'phone_syms_' are a set of phone-ids, typically 1, 2, .. num_phones.
|
||||
kaldi::ConstIntegerSet<Label> phone_syms_;
|
||||
|
||||
// disambig_syms_ is the set of integer ids of the disambiguation symbols,
|
||||
// usually represented in text form as #0, #1, #2, etc. These are inserted
|
||||
// into the grammar (for #0) and the lexicon (for #1, #2, ...) in order to
|
||||
// make the composed FSTs determinizable. They are treated "specially" by the
|
||||
// context FST in that they are not part of the context, they are just "passed
|
||||
// through" via self-loops. See the Mohri chapter mrentioned above for more
|
||||
// information.
|
||||
kaldi::ConstIntegerSet<Label> disambig_syms_;
|
||||
|
||||
// subsequential_symbol_, represented as "$" in the Mohri chapter mentioned
|
||||
// above, is something which terminates phonetic sequences to force out the
|
||||
// last phones-in-context. In our implementation it's added to det(LG) as a
|
||||
// self-loop on final states before composing with C.
|
||||
// (c.f. AddSubsequentialLoop()).
|
||||
Label subsequential_symbol_;
|
||||
|
||||
|
||||
// pseudo_eps_symbol_, which in printed form we refer to as "#-1", is a symbol that
|
||||
// appears on the ilabels of the context transducer C, i.e. the olabels of this
|
||||
// FST which is C's inverse. It is a symbol we introduce to solve a special problem
|
||||
// in systems with right-context (context_width_ > central_position_ + 1) that
|
||||
// use disambiguation symbols. It exists to prevent CLG from being nondeterminizable.
|
||||
//
|
||||
// The issue is that, in this case, the disambiguation symbols are shifted
|
||||
// left w.r.t. the phones, and there becomes an ambiguity, if a disambiguation
|
||||
// symbol appears at the start of a sequence on the input of CLG, about
|
||||
// whether it was at the very start of the input of LG, or just after, say,
|
||||
// the first real phone. This can lead to determinization failure under
|
||||
// certain circumstances. What we do if we need pseudo_eps_symbol_ to be not
|
||||
// epsilon, we create a special symbol with symbol-id 1 and sequence
|
||||
// representation (ilabels entry) [ 0 ] .
|
||||
int32 pseudo_eps_symbol_;
|
||||
|
||||
// maps from vector<int32>, representing phonetic contexts of length
|
||||
// context_width_ - 1, to StateId. (The states of the "C" fst correspond to
|
||||
// phonetic contexts, but we only create them as and when they are needed).
|
||||
VectorToStateMap state_map_;
|
||||
|
||||
// The inverse of 'state_map_': gives us the phonetic context corresponding to
|
||||
// each state-id.
|
||||
std::vector<std::vector<int32> > state_seqs_;
|
||||
|
||||
// maps from vector<int32>, representing phonetic contexts of length
|
||||
// context_width_ - 1, to Label. These are actually the output labels of this
|
||||
// InverseContextFst (because of the "Inverse" part), but for historical
|
||||
// reasons and because we've used the term ilabels" in the documentation, we
|
||||
// still call these "ilabels").
|
||||
VectorToLabelMap ilabel_map_;
|
||||
|
||||
// ilabel_info_ is the reverse map of ilabel_map_.
|
||||
// Indexed by olabel (although we call this ilabel_info_ for historical
|
||||
// reasons and because is for the ilabels of C), ilabel_info_[i] gives
|
||||
// information about the meaning of each symbol on the input of C
|
||||
// aka the output of inv(C).
|
||||
// See "http://kaldi-asr.org/doc/tree_externals.html#tree_ilabel".
|
||||
std::vector<std::vector<int32> > ilabel_info_;
|
||||
|
||||
};
|
||||
|
||||
} // namespace fst
|
||||
|
||||
|
||||
#endif // KALDI_FSTEXT_CONTEXT_FST_H_
|
||||
@@ -0,0 +1,512 @@
|
||||
// fstext/deterministic-fst-inl.h
|
||||
|
||||
// Copyright 2011-2012 Gilles Boulianne
|
||||
// 2014 Telepoint Global Hosting Service, LLC. (Author: David Snyder)
|
||||
// 2012-2015 Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_DETERMINISTIC_FST_INL_H_
|
||||
#define KALDI_FSTEXT_DETERMINISTIC_FST_INL_H_
|
||||
#include "base/kaldi-common.h"
|
||||
#include "fstext/fstext-utils.h"
|
||||
|
||||
|
||||
namespace fst {
|
||||
// Do not include this file directly. It is included by deterministic-fst.h.
|
||||
|
||||
template<class Arc>
|
||||
typename Arc::StateId
|
||||
BackoffDeterministicOnDemandFst<Arc>::GetBackoffState(StateId s,
|
||||
Weight *w) {
|
||||
ArcIterator<Fst<Arc> > aiter(fst_, s);
|
||||
if (aiter.Done()) // no arcs.
|
||||
return kNoStateId;
|
||||
const Arc &arc = aiter.Value();
|
||||
if (arc.ilabel == 0) {
|
||||
*w = arc.weight;
|
||||
return arc.nextstate;
|
||||
} else {
|
||||
return kNoStateId;
|
||||
}
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
typename Arc::Weight BackoffDeterministicOnDemandFst<Arc>::Final(StateId state) {
|
||||
Weight w = fst_.Final(state);
|
||||
if (w != Weight::Zero()) return w;
|
||||
Weight backoff_w;
|
||||
StateId backoff_state = GetBackoffState(state, &backoff_w);
|
||||
if (backoff_state == kNoStateId) return Weight::Zero();
|
||||
else return Times(backoff_w, this->Final(backoff_state));
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
BackoffDeterministicOnDemandFst<Arc>::BackoffDeterministicOnDemandFst(
|
||||
const Fst<Arc> &fst): fst_(fst) {
|
||||
#ifdef KALDI_PARANOID
|
||||
KALDI_ASSERT(fst_.Properties(kILabelSorted|kIDeterministic, true) ==
|
||||
(kILabelSorted|kIDeterministic) &&
|
||||
"Input FST is not i-label sorted and deterministic.");
|
||||
#endif
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
bool BackoffDeterministicOnDemandFst<Arc>::GetArc(
|
||||
StateId s, Label ilabel, Arc *oarc) {
|
||||
KALDI_ASSERT(ilabel != 0); // We don't allow GetArc for epsilon.
|
||||
|
||||
SortedMatcher<Fst<Arc> > sm(fst_, MATCH_INPUT, 1);
|
||||
sm.SetState(s);
|
||||
if (sm.Find(ilabel)) {
|
||||
const Arc &arc = sm.Value();
|
||||
*oarc = arc;
|
||||
return true;
|
||||
} else {
|
||||
Weight backoff_w;
|
||||
StateId backoff_state = GetBackoffState(s, &backoff_w);
|
||||
if (backoff_state == kNoStateId) return false;
|
||||
if (!this->GetArc(backoff_state, ilabel, oarc)) return false;
|
||||
oarc->weight = Times(oarc->weight, backoff_w);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
UnweightedNgramFst<Arc>::UnweightedNgramFst(int n): n_(n) {
|
||||
// Starting state is an empty vector
|
||||
std::vector<Label> start_state;
|
||||
state_vec_.push_back(start_state);
|
||||
start_state_ = 0;
|
||||
state_map_[start_state] = 0;
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
bool UnweightedNgramFst<Arc>::GetArc(
|
||||
StateId s, Label ilabel, Arc *oarc) {
|
||||
|
||||
// The state ids increment with each state we encounter.
|
||||
// if the assert fails, then we are trying to access
|
||||
// unseen states that are not immediately traversable.
|
||||
KALDI_ASSERT(static_cast<size_t>(s) < state_vec_.size());
|
||||
std::vector<Label> seq = state_vec_[s];
|
||||
// Update state info.
|
||||
seq.push_back(ilabel);
|
||||
if (seq.size() > n_-1) {
|
||||
// Remove oldest word in the history.
|
||||
seq.erase(seq.begin());
|
||||
}
|
||||
std::pair<const std::vector<Label>, StateId> new_state(
|
||||
seq,
|
||||
static_cast<Label>(state_vec_.size()));
|
||||
// Now get state id for destination state.
|
||||
typedef typename MapType::iterator IterType;
|
||||
std::pair<IterType, bool> result = state_map_.insert(new_state);
|
||||
if (result.second == true) {
|
||||
state_vec_.push_back(seq);
|
||||
}
|
||||
oarc->weight = Weight::One(); // Because the FST is unweightd.
|
||||
oarc->ilabel = ilabel;
|
||||
oarc->olabel = ilabel;
|
||||
oarc->nextstate = result.first->second; // The next state id.
|
||||
// All arcs can be matched.
|
||||
return true;
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
typename Arc::Weight UnweightedNgramFst<Arc>::Final(StateId state) {
|
||||
KALDI_ASSERT(state < static_cast<StateId>(state_vec_.size()));
|
||||
return Weight::One();
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
ComposeDeterministicOnDemandFst<Arc>::ComposeDeterministicOnDemandFst(
|
||||
DeterministicOnDemandFst<Arc> *fst1,
|
||||
DeterministicOnDemandFst<Arc> *fst2): fst1_(fst1), fst2_(fst2) {
|
||||
KALDI_ASSERT(fst1 != NULL && fst2 != NULL);
|
||||
if (fst1_->Start() == -1 || fst2_->Start() == -1) {
|
||||
start_state_ = -1;
|
||||
next_state_ = 0; // actually we don't care about this value.
|
||||
} else {
|
||||
start_state_ = 0;
|
||||
std::pair<StateId,StateId> start_pair(fst1_->Start(), fst2_->Start());
|
||||
state_map_[start_pair] = start_state_;
|
||||
state_vec_.push_back(start_pair);
|
||||
next_state_ = 1;
|
||||
}
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
typename Arc::Weight ComposeDeterministicOnDemandFst<Arc>::Final(StateId s) {
|
||||
KALDI_ASSERT(s < static_cast<StateId>(state_vec_.size()));
|
||||
const std::pair<StateId, StateId> &pr (state_vec_[s]);
|
||||
return Times(fst1_->Final(pr.first), fst2_->Final(pr.second));
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
bool ComposeDeterministicOnDemandFst<Arc>::GetArc(StateId s, Label ilabel,
|
||||
Arc *oarc) {
|
||||
typedef typename MapType::iterator IterType;
|
||||
KALDI_ASSERT(ilabel != 0 &&
|
||||
"This program expects epsilon-free compact lattices as input");
|
||||
KALDI_ASSERT(s < static_cast<StateId>(state_vec_.size()));
|
||||
const std::pair<StateId, StateId> pr (state_vec_[s]);
|
||||
|
||||
Arc arc1;
|
||||
if (!fst1_->GetArc(pr.first, ilabel, &arc1)) return false;
|
||||
if (arc1.olabel == 0) { // There is no output label on the
|
||||
// arc, so only the first state changes.
|
||||
std::pair<const std::pair<StateId, StateId>, StateId> new_value(
|
||||
std::pair<StateId, StateId>(arc1.nextstate, pr.second),
|
||||
next_state_);
|
||||
|
||||
std::pair<IterType, bool> result = state_map_.insert(new_value);
|
||||
oarc->ilabel = ilabel;
|
||||
oarc->olabel = 0;
|
||||
oarc->nextstate = result.first->second;
|
||||
oarc->weight = arc1.weight;
|
||||
if (result.second == true) { // was inserted
|
||||
next_state_++;
|
||||
const std::pair<StateId, StateId> &new_pair (new_value.first);
|
||||
state_vec_.push_back(new_pair);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
// There is an output label, so we need to traverse an arc on the
|
||||
// second fst also.
|
||||
Arc arc2;
|
||||
if (!fst2_->GetArc(pr.second, arc1.olabel, &arc2)) return false;
|
||||
std::pair<const std::pair<StateId, StateId>, StateId> new_value(
|
||||
std::pair<StateId, StateId>(arc1.nextstate, arc2.nextstate),
|
||||
next_state_);
|
||||
std::pair<IterType, bool> result =
|
||||
state_map_.insert(new_value);
|
||||
oarc->ilabel = ilabel;
|
||||
oarc->olabel = arc2.olabel;
|
||||
oarc->nextstate = result.first->second;
|
||||
oarc->weight = Times(arc1.weight, arc2.weight);
|
||||
if (result.second == true) { // was inserted
|
||||
next_state_++;
|
||||
const std::pair<StateId, StateId> &new_pair (new_value.first);
|
||||
state_vec_.push_back(new_pair);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
inline size_t CacheDeterministicOnDemandFst<Arc>::GetIndex(
|
||||
StateId src_state, Label ilabel) {
|
||||
const StateId p1 = 26597, p2 = 50329; // these are two
|
||||
// values that I drew at random from a table of primes.
|
||||
// note: num_cached_arcs_ > 0.
|
||||
|
||||
// We cast to size_t before the modulus, to ensure the
|
||||
// result is positive.
|
||||
return static_cast<size_t>(src_state * p1 + ilabel * p2) %
|
||||
static_cast<size_t>(num_cached_arcs_);
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
CacheDeterministicOnDemandFst<Arc>::CacheDeterministicOnDemandFst(
|
||||
DeterministicOnDemandFst<Arc> *fst,
|
||||
StateId num_cached_arcs): fst_(fst),
|
||||
num_cached_arcs_(num_cached_arcs),
|
||||
cached_arcs_(num_cached_arcs) {
|
||||
KALDI_ASSERT(num_cached_arcs > 0);
|
||||
for (StateId i = 0; i < num_cached_arcs; i++)
|
||||
cached_arcs_[i].first = kNoStateId; // Invalidate all elements of the cache.
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
bool CacheDeterministicOnDemandFst<Arc>::GetArc(StateId s, Label ilabel,
|
||||
Arc *oarc) {
|
||||
// Note: we don't cache anything in case a requested arc does not exist.
|
||||
// In the uses that we imagine this will be put to, essentially all the
|
||||
// requested arcs will exist. This only affects efficiency.
|
||||
KALDI_ASSERT(s >= 0 && ilabel != 0);
|
||||
size_t index = this->GetIndex(s, ilabel);
|
||||
if (cached_arcs_[index].first == s &&
|
||||
cached_arcs_[index].second.ilabel == ilabel) {
|
||||
*oarc = cached_arcs_[index].second;
|
||||
return true;
|
||||
} else {
|
||||
Arc arc;
|
||||
if (fst_->GetArc(s, ilabel, &arc)) {
|
||||
cached_arcs_[index].first = s;
|
||||
cached_arcs_[index].second = arc;
|
||||
*oarc = arc;
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
LmExampleDeterministicOnDemandFst<Arc>::LmExampleDeterministicOnDemandFst(
|
||||
void *lm, Label bos_symbol, Label eos_symbol):
|
||||
lm_(lm), bos_symbol_(bos_symbol), eos_symbol_(eos_symbol) {
|
||||
std::vector<Label> begin_state; // history state corresponding to beginning of sentence
|
||||
begin_state.push_back(bos_symbol); // Depending how your LM is set up, you might
|
||||
// want to have a history vector with more than one bos_symbol on it.
|
||||
|
||||
state_vec_.push_back(begin_state);
|
||||
start_state_ = 0;
|
||||
state_map_[begin_state] = 0;
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
typename Arc::Weight LmExampleDeterministicOnDemandFst<Arc>::Final(StateId s) {
|
||||
KALDI_ASSERT(static_cast<size_t>(s) < state_vec_.size());
|
||||
// In a real version you would probably use the following variable somehow
|
||||
// (commenting it because it's generating warnings).
|
||||
// const std::vector<Label> &wseq = state_vec_[s];
|
||||
float log_prob = -0.5; // e.g. log_prob = lm->GetLogProb(wseq, eos_symbol_);
|
||||
return Weight(-log_prob); // assuming weight is FloatWeight.
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
bool LmExampleDeterministicOnDemandFst<Arc>::GetArc(
|
||||
StateId s, Label ilabel, Arc *oarc) {
|
||||
KALDI_ASSERT(static_cast<size_t>(s) < state_vec_.size());
|
||||
std::vector<Label> wseq = state_vec_[s];
|
||||
float log_prob = -0.25; // e.g. log_prob = lm->GetLogProb(wseq, ilabel);
|
||||
wseq.push_back(ilabel); // the code might be different if your histories are the
|
||||
// other way around.
|
||||
|
||||
while (0) { // e.g. while !lm->HistoryStateExists(wseq)
|
||||
wseq.erase(wseq.begin(), wseq.begin() + 1); // remove most distant element of history.
|
||||
// note: if your histories are the other way round, you might just do
|
||||
// wseq.pop() here.
|
||||
}
|
||||
if (log_prob == -std::numeric_limits<float>::infinity()) { // assume this
|
||||
// is what happens if prob of the word is zero. Some LMs will never
|
||||
// return zero.
|
||||
return false; // no arc.
|
||||
}
|
||||
std::pair<const std::vector<Label>, StateId> new_value(
|
||||
wseq,
|
||||
static_cast<Label>(state_vec_.size()));
|
||||
|
||||
// Now get state id for destination state.
|
||||
typedef typename MapType::iterator IterType;
|
||||
std::pair<IterType, bool> result = state_map_.insert(new_value);
|
||||
if (result.second == true) // was inserted
|
||||
state_vec_.push_back(wseq);
|
||||
oarc->ilabel = ilabel;
|
||||
oarc->olabel = ilabel;
|
||||
oarc->nextstate = result.first->second; // the next-state id.
|
||||
oarc->weight = Weight(-log_prob);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
template<class Arc>
|
||||
void ComposeDeterministicOnDemand(const Fst<Arc> &fst1,
|
||||
DeterministicOnDemandFst<Arc> *fst2,
|
||||
MutableFst<Arc> *fst_composed) {
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef std::pair<StateId, StateId> StatePair;
|
||||
typedef unordered_map<StatePair, StateId,
|
||||
kaldi::PairHasher<StateId> > MapType;
|
||||
typedef typename MapType::iterator IterType;
|
||||
|
||||
fst_composed->DeleteStates();
|
||||
|
||||
MapType state_map;
|
||||
std::queue<StatePair> state_queue;
|
||||
|
||||
// Set start state in fst_composed.
|
||||
StateId s1 = fst1.Start(),
|
||||
s2 = fst2->Start(),
|
||||
start_state = fst_composed->AddState();
|
||||
StatePair start_pair(s1, s2);
|
||||
state_queue.push(start_pair);
|
||||
fst_composed->SetStart(start_state);
|
||||
// A mapping between pairs of states in fst1 and fst2 and the corresponding
|
||||
// state in fst_composed.
|
||||
std::pair<const StatePair, StateId> start_map(start_pair, start_state);
|
||||
std::pair<IterType, bool> result = state_map.insert(start_map);
|
||||
KALDI_ASSERT(result.second == true);
|
||||
|
||||
while (!state_queue.empty()) {
|
||||
StatePair q = state_queue.front();
|
||||
StateId q1 = q.first,
|
||||
q2 = q.second;
|
||||
state_queue.pop();
|
||||
// If the product of the final weights of the two fsts is non-zero then
|
||||
// we can set a final-prob in fst_composed
|
||||
Weight final_weight = Times(fst1.Final(q1), fst2->Final(q2));
|
||||
if (final_weight != Weight::Zero()) {
|
||||
KALDI_ASSERT(state_map.find(q) != state_map.end());
|
||||
fst_composed->SetFinal(state_map[q], final_weight);
|
||||
}
|
||||
|
||||
// for each pair of edges from fst1 and fst2 at q1 and q2.
|
||||
for (ArcIterator<Fst<Arc> > aiter(fst1, q1); !aiter.Done(); aiter.Next()) {
|
||||
const Arc &arc1 = aiter.Value();
|
||||
Arc arc2;
|
||||
StatePair next_pair;
|
||||
StateId next_state1 = arc1.nextstate,
|
||||
next_state2,
|
||||
next_state;
|
||||
// If there is an epsilon on the arc of fst1 we transition to the next
|
||||
// state but keep fst2 at the current state.
|
||||
if (arc1.olabel == 0) {
|
||||
next_state2 = q2;
|
||||
} else {
|
||||
bool match = fst2->GetArc(q2, arc1.olabel, &arc2);
|
||||
if (!match) // There is no matching arc -> nothing to do.
|
||||
continue;
|
||||
next_state2 = arc2.nextstate;
|
||||
}
|
||||
next_pair = StatePair(next_state1, next_state2);
|
||||
IterType sitr = state_map.find(next_pair);
|
||||
// If sitr == state_map.end() then the state isn't in fst_composed yet.
|
||||
if (sitr == state_map.end()) {
|
||||
next_state = fst_composed->AddState();
|
||||
std::pair<const StatePair, StateId> new_state(
|
||||
next_pair, next_state);
|
||||
std::pair<IterType, bool> result = state_map.insert(new_state);
|
||||
// Since we already checked if state_map contained new_state,
|
||||
// it should always be added if we reach here.
|
||||
KALDI_ASSERT(result.second == true);
|
||||
state_queue.push(next_pair);
|
||||
// If sitr != state_map.end() then the next state is already in
|
||||
// the state_map.
|
||||
} else {
|
||||
next_state = sitr->second;
|
||||
}
|
||||
if (arc1.olabel == 0) {
|
||||
fst_composed->AddArc(state_map[q], Arc(arc1.ilabel, 0, arc1.weight,
|
||||
next_state));
|
||||
} else {
|
||||
fst_composed->AddArc(state_map[q], Arc(arc1.ilabel, arc2.olabel,
|
||||
Times(arc1.weight, arc2.weight), next_state));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// we are doing *fst_composed = Compose(Inverse(*left), right).
|
||||
template<class Arc>
|
||||
void ComposeDeterministicOnDemandInverse(const Fst<Arc> &right,
|
||||
DeterministicOnDemandFst<Arc> *left,
|
||||
MutableFst<Arc> *fst_composed) {
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef std::pair<StateId, StateId> StatePair;
|
||||
typedef unordered_map<StatePair, StateId,
|
||||
kaldi::PairHasher<StateId> > MapType;
|
||||
typedef typename MapType::iterator IterType;
|
||||
|
||||
fst_composed->DeleteStates();
|
||||
|
||||
// the queue and map contain pairs (state-in-left, state-in-right)
|
||||
MapType state_map;
|
||||
std::queue<StatePair> state_queue;
|
||||
|
||||
// Set start state in fst_composed.
|
||||
StateId s_left = left->Start(),
|
||||
s_right = right.Start();
|
||||
if (s_left == kNoStateId || s_right == kNoStateId)
|
||||
return; // Empty result.
|
||||
StatePair start_pair(s_left, s_right);
|
||||
StateId start_state = fst_composed->AddState();
|
||||
state_queue.push(start_pair);
|
||||
fst_composed->SetStart(start_state);
|
||||
// A mapping between pairs of states in *left and right, and the corresponding
|
||||
// state in fst_composed.
|
||||
std::pair<const StatePair, StateId> start_map(start_pair, start_state);
|
||||
std::pair<IterType, bool> result = state_map.insert(start_map);
|
||||
KALDI_ASSERT(result.second == true);
|
||||
|
||||
while (!state_queue.empty()) {
|
||||
StatePair q = state_queue.front();
|
||||
StateId q_left = q.first,
|
||||
q_right = q.second;
|
||||
state_queue.pop();
|
||||
// If the product of the final weights of the two fsts is non-zero then
|
||||
// we can set a final-prob in fst_composed
|
||||
Weight final_weight = Times(left->Final(q_left), right.Final(q_right));
|
||||
if (final_weight != Weight::Zero()) {
|
||||
KALDI_ASSERT(state_map.find(q) != state_map.end());
|
||||
fst_composed->SetFinal(state_map[q], final_weight);
|
||||
}
|
||||
|
||||
for (ArcIterator<Fst<Arc> > aiter(right, q_right); !aiter.Done(); aiter.Next()) {
|
||||
const Arc &arc_right = aiter.Value();
|
||||
Arc arc_left;
|
||||
StatePair next_pair;
|
||||
StateId next_state_right = arc_right.nextstate,
|
||||
next_state_left,
|
||||
next_state;
|
||||
// If there is an epsilon on the input side of the rigth arc, we
|
||||
// transition to the next state of the output but keep 'left' at the
|
||||
// current state.
|
||||
if (arc_right.ilabel == 0) {
|
||||
next_state_left = q_left;
|
||||
} else {
|
||||
bool match = left->GetArc(q_left, arc_right.ilabel, &arc_left);
|
||||
if (!match) // There is no matching arc -> nothing to do.
|
||||
continue;
|
||||
// the next 'swap' is because we are composing with the inverse of
|
||||
// *left. Just removing the swap statement wouldn't let us compose
|
||||
// with non-inverted *left though, because the GetArc function call
|
||||
// above interprets the second argument as an ilabel not an olabel.
|
||||
std::swap(arc_left.ilabel, arc_left.olabel);
|
||||
next_state_left = arc_left.nextstate;
|
||||
}
|
||||
next_pair = StatePair(next_state_left, next_state_right);
|
||||
IterType sitr = state_map.find(next_pair);
|
||||
// If sitr == state_map.end() then the state isn't in fst_composed yet.
|
||||
if (sitr == state_map.end()) {
|
||||
next_state = fst_composed->AddState();
|
||||
std::pair<const StatePair, StateId> new_state(
|
||||
next_pair, next_state);
|
||||
std::pair<IterType, bool> result = state_map.insert(new_state);
|
||||
// Since we already checked if state_map contained new_state,
|
||||
// it should always be added if we reach here.
|
||||
KALDI_ASSERT(result.second == true);
|
||||
state_queue.push(next_pair);
|
||||
// If sitr != state_map.end() then the next state is already in
|
||||
// the state_map.
|
||||
} else {
|
||||
next_state = sitr->second;
|
||||
}
|
||||
if (arc_right.ilabel == 0) {
|
||||
// we didn't get an actual arc from the left FST.
|
||||
fst_composed->AddArc(state_map[q], Arc(0, arc_right.olabel,
|
||||
arc_right.weight,
|
||||
next_state));
|
||||
} else {
|
||||
fst_composed->AddArc(state_map[q],
|
||||
Arc(arc_left.ilabel, arc_right.olabel,
|
||||
Times(arc_left.weight, arc_right.weight),
|
||||
next_state));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,232 @@
|
||||
// fstext/deterministic-fst-test.cc
|
||||
|
||||
// Copyright 2009-2011 Gilles Boulianne
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fstext/deterministic-fst.h"
|
||||
#include "fstext/fst-test-utils.h"
|
||||
#include "util/kaldi-io.h"
|
||||
|
||||
#include <sys/stat.h>
|
||||
|
||||
namespace fst {
|
||||
using std::cout;
|
||||
using std::cerr;
|
||||
using std::endl;
|
||||
|
||||
bool FileExists(std::string strFilename) {
|
||||
struct stat stFileInfo;
|
||||
bool blnReturn;
|
||||
int intStat;
|
||||
|
||||
// Attempt to get the file attributes
|
||||
intStat = stat(strFilename.c_str(), &stFileInfo);
|
||||
if (intStat == 0) {
|
||||
// We were able to get the file attributes
|
||||
// so the file obviously exists.
|
||||
blnReturn = true;
|
||||
} else {
|
||||
// We were not able to get the file attributes.
|
||||
// This may mean that we don't have permission to
|
||||
// access the folder which contains this file. If you
|
||||
// need to do that level of checking, lookup the
|
||||
// return values of stat which will give you
|
||||
// more details on why stat failed.
|
||||
blnReturn = false;
|
||||
}
|
||||
|
||||
return blnReturn;
|
||||
}
|
||||
|
||||
// Simplify writing
|
||||
typedef fst::StdArc StdArc;
|
||||
typedef fst::StdArc::Label Label;
|
||||
typedef fst::StdArc::StateId StateId;
|
||||
typedef fst::StdVectorFst StdVectorFst;
|
||||
typedef fst::StdArc::Weight Weight;
|
||||
|
||||
|
||||
// something that looks like a language model FST with epsilon backoffs
|
||||
StdVectorFst* CreateBackoffFst() {
|
||||
StdVectorFst *fst = new StdVectorFst();
|
||||
fst->AddState(); // state 0
|
||||
fst->SetStart(0);
|
||||
fst->AddArc(0, StdArc(10, 10, 0.0, 1));
|
||||
|
||||
fst->AddState(); // state 1
|
||||
fst->AddArc(1, StdArc(12, 12, 0.0, 4));
|
||||
fst->AddArc(1, StdArc(0,0, 0.1,2)); // backoff from 1 to 2
|
||||
|
||||
fst->AddState(); // state 2
|
||||
fst->AddArc(2, StdArc(13, 13, 0.2, 4));
|
||||
fst->AddArc(2, StdArc(0,0, 0.3,3)); // backoff from 2 to 3
|
||||
|
||||
fst->AddState(); // state 3
|
||||
fst->AddArc(3, StdArc(14, 14, 0.4, 4));
|
||||
|
||||
fst->AddState(); // state 4
|
||||
fst->AddArc(4, StdArc(15, 15, 0.5, 5));
|
||||
|
||||
fst->AddState(); // state 5
|
||||
fst->SetFinal(5, 0.6);
|
||||
|
||||
return fst;
|
||||
}
|
||||
|
||||
// what the resulting DeterministicOnDemand FST should be like
|
||||
StdVectorFst* CreateResultFst() {
|
||||
StdVectorFst *fst = new StdVectorFst();
|
||||
fst->AddState(); // state 0
|
||||
fst->SetStart(0);
|
||||
fst->AddArc(0, StdArc(10, 10, 0.0, 1));
|
||||
|
||||
fst->AddState(); // state 1
|
||||
fst->AddArc(1, StdArc(12, 12, 0.0, 4));
|
||||
fst->AddArc(1, StdArc(13,13,0.3,4)); // went through 1 backoff
|
||||
fst->AddArc(1, StdArc(14,14,0.8,4)); // went through 2 backoffs
|
||||
|
||||
fst->AddState(); // state 2
|
||||
fst->AddState(); // state 3
|
||||
|
||||
fst->AddState(); // state 4
|
||||
fst->AddArc(4, StdArc(15, 15, 0.5, 5));
|
||||
|
||||
fst->AddState(); // state 5
|
||||
fst->SetFinal(5, 0.6);
|
||||
|
||||
return fst;
|
||||
}
|
||||
|
||||
void DeleteTestFst(StdVectorFst *fst) {
|
||||
delete fst;
|
||||
}
|
||||
|
||||
// Follow paths from an input fst representing a string
|
||||
// (poor man's composition)
|
||||
Weight WalkSinglePath(StdVectorFst *ifst, DeterministicOnDemandFst<StdArc> *dfst) {
|
||||
StdArc oarc; // = new StdArc();
|
||||
StateId isrc=ifst->Start();
|
||||
StateId dsrc=dfst->Start();
|
||||
Weight totalCost = Weight::One();
|
||||
|
||||
while (ifst->Final(isrc) == Weight::Zero()) { // while not final
|
||||
fst::ArcIterator<StdVectorFst> aiter(*ifst, isrc);
|
||||
const StdArc &iarc = aiter.Value();
|
||||
if (dfst->GetArc(dsrc, iarc.olabel, &oarc)) {
|
||||
Weight cost = Times(iarc.weight, oarc.weight);
|
||||
// cout << " Matched label "<<iarc.olabel<<" at summed cost "<<cost<<endl;
|
||||
totalCost = Times(totalCost, cost);
|
||||
} else {
|
||||
cout << " Can't match arc ["<<iarc.ilabel<<","<<iarc.olabel<<","<<iarc.weight<<"] from "<<isrc<<endl;
|
||||
exit(1);
|
||||
}
|
||||
isrc = iarc.nextstate;
|
||||
KALDI_LOG << "Setting dsrc = " << oarc.nextstate;
|
||||
dsrc = oarc.nextstate;
|
||||
}
|
||||
totalCost = Times(totalCost, dfst->Final(dsrc));
|
||||
|
||||
cout << " Total cost: " << totalCost << endl;
|
||||
return totalCost;
|
||||
}
|
||||
|
||||
|
||||
void TestBackoffAndCache() {
|
||||
// Build from existing fst
|
||||
cout << "Test with single generated backoff FST" << endl;
|
||||
StdVectorFst *nfst = CreateBackoffFst();
|
||||
StdVectorFst *rfst = CreateResultFst();
|
||||
|
||||
// before using, make sure that it is input sorted
|
||||
ArcSort(nfst, StdILabelCompare());
|
||||
BackoffDeterministicOnDemandFst<StdArc> dfst1a(*nfst);
|
||||
CacheDeterministicOnDemandFst<StdArc> dfst1(&dfst1a);
|
||||
|
||||
// Compare all arcs in dfst1 with expected result
|
||||
for (StateIterator<StdVectorFst> riter(*rfst); !riter.Done(); riter.Next()) {
|
||||
StateId rsrc = riter.Value();
|
||||
// verify that states have same weight (or final status)
|
||||
assert(ApproxEqual(rfst->Final(rsrc), dfst1.Final(rsrc)));
|
||||
for (ArcIterator<StdVectorFst> aiter(*rfst, rsrc); !aiter.Done(); aiter.Next()) {
|
||||
StdArc rarc = aiter.Value();
|
||||
StdArc darc;
|
||||
if (dfst1.GetArc(rsrc, rarc.ilabel, &darc)) {
|
||||
assert(ApproxEqual(rarc.weight, darc.weight, 0.001));
|
||||
assert(rarc.ilabel==darc.ilabel);
|
||||
assert(rarc.olabel==darc.olabel);
|
||||
assert(rarc.nextstate == darc.nextstate);
|
||||
cerr << " Got same arc at state "<<rsrc<<": "<<rarc.ilabel<<" "<<darc.ilabel<<endl;
|
||||
} else {
|
||||
cerr << "Couldn't find arc "<<rarc.ilabel<<" for state "<<rsrc<<endl;
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
delete nfst;
|
||||
delete rfst;
|
||||
}
|
||||
|
||||
void TestCompose() {
|
||||
cout << "Test with single generated backoff FST" << endl;
|
||||
StdVectorFst *nfst = CreateBackoffFst();
|
||||
StdVectorFst *rfst = CreateResultFst();
|
||||
|
||||
StdVectorFst composed_fst;
|
||||
Compose(*rfst, *rfst, &composed_fst);
|
||||
|
||||
// before using, make sure that it is input sorted
|
||||
ArcSort(nfst, StdILabelCompare());
|
||||
BackoffDeterministicOnDemandFst<StdArc> dfst1a(*nfst);
|
||||
ComposeDeterministicOnDemandFst<StdArc> dfst1b(&dfst1a, &dfst1a);
|
||||
CacheDeterministicOnDemandFst<StdArc> dfst1(&dfst1b);
|
||||
|
||||
typedef StdArc::StateId StateId;
|
||||
std::map<StateId, StateId> state_map;
|
||||
state_map[composed_fst.Start()] = dfst1.Start();
|
||||
|
||||
VectorFst<StdArc> path_fst;
|
||||
ShortestPath(composed_fst, &path_fst);
|
||||
|
||||
BackoffDeterministicOnDemandFst<StdArc> dfst2(composed_fst);
|
||||
|
||||
Weight w1 = WalkSinglePath(&path_fst, &dfst1),
|
||||
w2 = WalkSinglePath(&path_fst, &dfst2);
|
||||
KALDI_ASSERT(ApproxEqual(w1, w2));
|
||||
|
||||
delete rfst;
|
||||
delete nfst;
|
||||
|
||||
{ // Mostly checking for compilation errors here.
|
||||
LmExampleDeterministicOnDemandFst<StdArc> lm_eg(NULL, 2, 3);
|
||||
KALDI_ASSERT(lm_eg.Start() == 0);
|
||||
KALDI_ASSERT(lm_eg.Final(0).Value() == 0.5); // I made it this value.
|
||||
StdArc arc;
|
||||
bool b = lm_eg.GetArc(0, 100, &arc);
|
||||
KALDI_ASSERT(b && arc.nextstate == 1 && arc.ilabel == 100 && arc.olabel == 100
|
||||
&& arc.weight.Value() == 0.25);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace fst;
|
||||
TestBackoffAndCache();
|
||||
TestCompose();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
// fstext/deterministic-fst.h
|
||||
|
||||
// Copyright 2011-2012 Gilles Boulianne
|
||||
// 2014 Telepoint Global Hosting Service, LLC. (Author: David Snyder)
|
||||
// 2012-2015 Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// This file includes material from the OpenFST Library v1.2.7 available at
|
||||
// http://www.openfst.org and released under the Apache License Version 2.0.
|
||||
//
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Copyright 2005-2010 Google, Inc.
|
||||
// Author: riley@google.com (Michael Riley)
|
||||
|
||||
#ifndef KALDI_FSTEXT_DETERMINISTIC_FST_H_
|
||||
#define KALDI_FSTEXT_DETERMINISTIC_FST_H_
|
||||
|
||||
/* This header defines the DeterministicOnDemand interface,
|
||||
which is an FST with a special interface that allows
|
||||
only a single arc with a non-epsilon input symbol
|
||||
out of each state.
|
||||
*/
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
|
||||
#include "util/stl-utils.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
/// \addtogroup deterministic_fst_group "Classes and functions related to on-demand deterministic FST's"
|
||||
/// @{
|
||||
|
||||
|
||||
/// class DeterministicOnDemandFst is an "FST-like" base-class. It does not
|
||||
/// actually inherit from any Fst class because its interface is not exactly the
|
||||
/// same; it's much smaller. It assumes that the FST can have only one arc for
|
||||
/// any given input symbol, which makes the GetArc function below possible.
|
||||
/// (The FST is also assumed to be free of input epsilons). Note: we don't use
|
||||
/// "const" in this interface, because it creates problems when we do things
|
||||
/// like caching.
|
||||
template<class Arc>
|
||||
class DeterministicOnDemandFst {
|
||||
public:
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef typename Arc::Label Label;
|
||||
|
||||
virtual StateId Start() = 0;
|
||||
|
||||
virtual Weight Final(StateId s) = 0;
|
||||
|
||||
/// Note: ilabel must not be epsilon.
|
||||
virtual bool GetArc(StateId s, Label ilabel, Arc *oarc) = 0;
|
||||
|
||||
virtual ~DeterministicOnDemandFst() { }
|
||||
};
|
||||
|
||||
/**
|
||||
This class wraps an Fst, representing a language model, using the interface
|
||||
for "BackoffDeterministicOnDemandFst". We expect that backoff arcs in the
|
||||
language model will have the epsilon label (label 0) on the arcs, and that
|
||||
there will be no other epsilons in the language model. We follow the epsilon
|
||||
arcs as long as a particular arc (or a final-prob) is not found at the
|
||||
current state.
|
||||
*/
|
||||
template<class Arc>
|
||||
class BackoffDeterministicOnDemandFst: public DeterministicOnDemandFst<Arc> {
|
||||
public:
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Label Label;
|
||||
|
||||
explicit BackoffDeterministicOnDemandFst(const Fst<Arc> &fst);
|
||||
|
||||
StateId Start() { return fst_.Start(); }
|
||||
|
||||
Weight Final(StateId s);
|
||||
|
||||
bool GetArc(StateId s, Label ilabel, Arc *oarc);
|
||||
|
||||
private:
|
||||
inline StateId GetBackoffState(StateId s, Weight *w);
|
||||
|
||||
const Fst<Arc> &fst_;
|
||||
};
|
||||
|
||||
/**
|
||||
Class ScaleDeterministicOnDemandFst takes another DeterministicOnDemandFst
|
||||
and scales the weights (like applying a language-model scale). For instance,
|
||||
to subtract existing LM scores from a lattice you could use this with
|
||||
a negative weight; and to interpolate LMs you can also use this with
|
||||
weights less than one.
|
||||
|
||||
It's specialized for StdArc because there is no generic way to scale weights.
|
||||
*/
|
||||
class ScaleDeterministicOnDemandFst: public DeterministicOnDemandFst<StdArc> {
|
||||
public:
|
||||
typedef StdArc::Weight Weight;
|
||||
typedef StdArc::StateId StateId;
|
||||
typedef StdArc::Label Label;
|
||||
|
||||
// Constructor does not take ownership of 'det_fst'.
|
||||
ScaleDeterministicOnDemandFst(float scale,
|
||||
DeterministicOnDemandFst<StdArc> *det_fst):
|
||||
scale_(scale), det_fst_(*det_fst) { }
|
||||
|
||||
StateId Start() { return det_fst_.Start(); }
|
||||
|
||||
Weight Final(StateId s) {
|
||||
// Note: Weight is indirectly a typedef to TropicalWeight.
|
||||
Weight final = det_fst_.Final(s);
|
||||
if (final == Weight::Zero()) return Weight::Zero();
|
||||
else return TropicalWeight(final.Value() * scale_);
|
||||
}
|
||||
|
||||
inline bool GetArc(StateId s, Label ilabel, StdArc *oarc) {
|
||||
if (det_fst_.GetArc(s, ilabel, oarc)) {
|
||||
oarc->weight = TropicalWeight(oarc->weight.Value() * scale_);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
float scale_;
|
||||
DeterministicOnDemandFst<StdArc> &det_fst_;
|
||||
};
|
||||
|
||||
/**
|
||||
The class UnweightedNgramFst is a DeterministicOnDemandFst whose states encode
|
||||
an n-gram history. Conceptually, for n-gram order n and k labels, the FST is an
|
||||
unweighted acceptor with about k^(n-1) states (ignoring end effects). However,
|
||||
the FST is created on demand and doesn't need the label vocabulary; GetArc
|
||||
matches on any input label. This class is primarily used together with
|
||||
ComposeDeterministicOnDemandFst to expand the n-gram history of lattices, ensuring
|
||||
that each arc has a sufficiently long unique word history.
|
||||
*/
|
||||
template<class Arc>
|
||||
class UnweightedNgramFst: public DeterministicOnDemandFst<Arc> {
|
||||
public:
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Label Label;
|
||||
|
||||
UnweightedNgramFst(int n);
|
||||
|
||||
StateId Start() { return start_state_; };
|
||||
|
||||
Weight Final(StateId s);
|
||||
|
||||
bool GetArc(StateId s, Label ilabel, Arc *oarc);
|
||||
|
||||
private:
|
||||
typedef unordered_map<std::vector<Label>,
|
||||
StateId, kaldi::VectorHasher<Label> > MapType;
|
||||
// The order of the n-gram.
|
||||
int n_;
|
||||
MapType state_map_;
|
||||
StateId start_state_;
|
||||
// Map from history-state to pair.
|
||||
std::vector<std::vector<Label> > state_vec_;
|
||||
};
|
||||
|
||||
template<class Arc>
|
||||
class ComposeDeterministicOnDemandFst: public DeterministicOnDemandFst<Arc> {
|
||||
public:
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef typename Arc::Label Label;
|
||||
|
||||
/// Note: constructor does not "take ownership" of the input fst's. The input
|
||||
/// fst's should be treated as const, in that their contents do not change,
|
||||
/// but they are not const as the DeterministicOnDemandFst's data-access
|
||||
/// functions are not const, for reasons relating to caching.
|
||||
ComposeDeterministicOnDemandFst(DeterministicOnDemandFst<Arc> *fst1,
|
||||
DeterministicOnDemandFst<Arc> *fst2);
|
||||
|
||||
virtual StateId Start() { return start_state_; }
|
||||
|
||||
virtual Weight Final(StateId s);
|
||||
|
||||
virtual bool GetArc(StateId s, Label ilabel, Arc *oarc);
|
||||
|
||||
private:
|
||||
DeterministicOnDemandFst<Arc> *fst1_;
|
||||
DeterministicOnDemandFst<Arc> *fst2_;
|
||||
typedef unordered_map<std::pair<StateId, StateId>, StateId, kaldi::PairHasher<StateId> > MapType;
|
||||
MapType state_map_;
|
||||
std::vector<std::pair<StateId, StateId> > state_vec_; // maps from
|
||||
// StateId to pair.
|
||||
StateId next_state_;
|
||||
StateId start_state_;
|
||||
};
|
||||
|
||||
template<class Arc>
|
||||
class CacheDeterministicOnDemandFst: public DeterministicOnDemandFst<Arc> {
|
||||
public:
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef typename Arc::Label Label;
|
||||
|
||||
/// We don't take ownership of this pointer. The argument is "really" const.
|
||||
CacheDeterministicOnDemandFst(DeterministicOnDemandFst<Arc> *fst,
|
||||
StateId num_cached_arcs = 100000);
|
||||
|
||||
virtual StateId Start() { return fst_->Start(); }
|
||||
|
||||
/// We don't bother caching the final-probs, just the arcs.
|
||||
virtual Weight Final(StateId s) { return fst_->Final(s); }
|
||||
|
||||
virtual bool GetArc(StateId s, Label ilabel, Arc *oarc);
|
||||
|
||||
private:
|
||||
// Get index for cached arc.
|
||||
inline size_t GetIndex(StateId src_state, Label ilabel);
|
||||
|
||||
DeterministicOnDemandFst<Arc> *fst_;
|
||||
StateId num_cached_arcs_;
|
||||
std::vector<std::pair<StateId, Arc> > cached_arcs_;
|
||||
};
|
||||
|
||||
|
||||
/// This class is for didactic purposes, it does not really do anything.
|
||||
/// It shows how you would wrap a language model. Note: you should probably
|
||||
/// have <s> and </s> not be real words in your LM, but <s> correspond somehow
|
||||
/// to the initial-state of the LM, and </s> be encoded in the final-probs.
|
||||
template<class Arc>
|
||||
class LmExampleDeterministicOnDemandFst: public DeterministicOnDemandFst<Arc> {
|
||||
public:
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef typename Arc::Label Label;
|
||||
|
||||
LmExampleDeterministicOnDemandFst(void *lm,
|
||||
Label bos_symbol,
|
||||
Label eos_symbol);
|
||||
|
||||
|
||||
virtual StateId Start() { return start_state_; }
|
||||
|
||||
/// We don't bother caching the final-probs, just the arcs.
|
||||
virtual Weight Final(StateId s);
|
||||
|
||||
virtual bool GetArc(StateId s, Label ilabel, Arc *oarc);
|
||||
|
||||
private:
|
||||
// Get index for cached arc.
|
||||
inline size_t GetIndex(StateId src_state, Label ilabel);
|
||||
|
||||
typedef unordered_map<std::vector<Label>, StateId, kaldi::VectorHasher<Label> > MapType;
|
||||
void *lm_;
|
||||
Label bos_symbol_; // beginning of sentence symbol
|
||||
Label eos_symbol_; // end of sentence symbol.
|
||||
// This example code does not handle <UNK>; we assume the LM has the same vocab as
|
||||
// the recognizer.
|
||||
MapType state_map_;
|
||||
StateId start_state_;
|
||||
std::vector<std::vector<Label> > state_vec_; // maps from history-state to pair.
|
||||
|
||||
void *lm; // wouldn't really be void.
|
||||
};
|
||||
|
||||
|
||||
// Compose an FST (which may be a lattice) with a DeterministicOnDemandFst and
|
||||
// store the result in fst_composed. This is mainly used for expanding lattice
|
||||
// n-gram histories, where fst1 is a lattice and fst2 is an UnweightedNgramFst.
|
||||
// This does not call Connect.
|
||||
template<class Arc>
|
||||
void ComposeDeterministicOnDemand(const Fst<Arc> &fst1,
|
||||
DeterministicOnDemandFst<Arc> *fst2,
|
||||
MutableFst<Arc> *fst_composed);
|
||||
|
||||
/**
|
||||
This function does
|
||||
'*fst_composed = Compose(Inverse(*fst2), fst1)'
|
||||
Note that the arguments are reversed; this is unfortunate but it's
|
||||
because the fst2 argument needs to be non-const and non-const arguments
|
||||
must follow const ones.
|
||||
This is the counterpart to ComposeDeterministicOnDemand, used for
|
||||
the case where the DeterministicOnDemandFst is on the left. The
|
||||
reason why we need to make the left-hand argument to compose the
|
||||
inverse of 'fst2' (i.e. with the input and output symbols swapped),
|
||||
is that the DeterministicOnDemandFst interface only supports lookup
|
||||
by ilabel (see its function GetArc).
|
||||
This does not call Connect().
|
||||
*/
|
||||
template<class Arc>
|
||||
void ComposeDeterministicOnDemandInverse(const Fst<Arc> &fst1,
|
||||
DeterministicOnDemandFst<Arc> *fst2,
|
||||
MutableFst<Arc> *fst_composed);
|
||||
|
||||
|
||||
|
||||
|
||||
/// @}
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#include "deterministic-fst-inl.h"
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,174 @@
|
||||
// fstext/determinize-lattice-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fstext/determinize-lattice.h"
|
||||
#include "fstext/lattice-utils.h"
|
||||
#include "fstext/fst-test-utils.h"
|
||||
#include "base/kaldi-math.h"
|
||||
|
||||
namespace fst {
|
||||
using std::vector;
|
||||
using std::cout;
|
||||
|
||||
void TestLatticeStringRepository() {
|
||||
typedef int32 IntType;
|
||||
|
||||
LatticeStringRepository<IntType> sr;
|
||||
typedef LatticeStringRepository<IntType>::Entry Entry;
|
||||
|
||||
for(int i = 0; i < 100; i++) {
|
||||
int len = kaldi::Rand() % 5;
|
||||
vector<IntType> str(len), str2(kaldi::Rand() % 4);
|
||||
const Entry *e = NULL;
|
||||
for(int i = 0; i < len; i++) {
|
||||
str[i] = kaldi::Rand() % 5;
|
||||
e = sr.Successor(e, str[i]);
|
||||
}
|
||||
sr.ConvertToVector(e, &str2);
|
||||
assert(str == str2);
|
||||
|
||||
int len2 = kaldi::Rand() % 5;
|
||||
str2.resize(len2);
|
||||
const Entry *f = sr.EmptyString(); // NULL
|
||||
for(int i = 0; i < len2; i++) {
|
||||
str2[i] = kaldi::Rand() % 5;
|
||||
f = sr.Successor(f, str2[i]);
|
||||
}
|
||||
vector<IntType> prefix, prefix2(kaldi::Rand() % 10),
|
||||
prefix3;
|
||||
for(int i = 0; i < len && i < len2; i++) {
|
||||
if (str[i] == str2[i]) prefix.push_back(str[i]);
|
||||
else break;
|
||||
}
|
||||
const Entry *g = sr.CommonPrefix(e, f);
|
||||
sr.ConvertToVector(g, &prefix2);
|
||||
sr.ConvertToVector(e, &prefix3);
|
||||
sr.ReduceToCommonPrefix(f, &prefix3);
|
||||
assert(prefix == prefix2);
|
||||
assert(prefix == prefix3);
|
||||
assert(sr.IsPrefixOf(g, e));
|
||||
assert(sr.IsPrefixOf(g, f));
|
||||
if (str.size() > prefix.size())
|
||||
assert(!sr.IsPrefixOf(e, g));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// test that determinization proceeds correctly on general
|
||||
// FSTs (not guaranteed determinzable, but we use the
|
||||
// max-states option to stop it getting out of control).
|
||||
template<class Arc> void TestDeterminizeLattice() {
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef int32 Int;
|
||||
typedef ArcTpl<CompactLatticeWeightTpl<Weight, Int> > CompactArc;
|
||||
|
||||
for(int i = 0; i < 100; i++) {
|
||||
RandFstOptions opts;
|
||||
opts.n_states = 4;
|
||||
opts.n_arcs = 10;
|
||||
opts.n_final = 2;
|
||||
opts.allow_empty = false;
|
||||
opts.weight_multiplier = 0.5; // impt for the randomly generated weights
|
||||
// to be exactly representable in float,
|
||||
// or this test fails because numerical differences can cause symmetry in
|
||||
// weights to be broken, which causes the wrong path to be chosen as far
|
||||
// as the string part is concerned.
|
||||
|
||||
VectorFst<Arc> *fst = RandFst<Arc>();
|
||||
std::cout << "FST before lattice-determinizing is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
VectorFst<Arc> det_fst;
|
||||
try {
|
||||
DeterminizeLatticeOptions lat_opts;
|
||||
lat_opts.max_mem = 100;
|
||||
|
||||
if (!DeterminizeLattice<TropicalWeight, int32>(*fst, &det_fst, lat_opts, NULL))
|
||||
throw std::runtime_error("could not determinize");
|
||||
std::cout << "FST after lattice-determinizing is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(det_fst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
assert(det_fst.Properties(kIDeterministic, true) & kIDeterministic);
|
||||
// OK, now determinize it a different way and check equivalence.
|
||||
// [note: it's not normal determinization, it's taking the best path
|
||||
// for any input-symbol sequence....
|
||||
VectorFst<CompactArc> compact_fst, compact_det_fst;
|
||||
ConvertLattice<Weight, Int>(*fst, &compact_fst, false);
|
||||
std::cout << "Compact FST is:\n";
|
||||
{
|
||||
FstPrinter<CompactArc> fstprinter(compact_fst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
if (kaldi::Rand() % 2 == 1)
|
||||
ConvertLattice<Weight, Int>(det_fst, &compact_det_fst, false);
|
||||
else
|
||||
if (!DeterminizeLattice<TropicalWeight, int32>(*fst, &compact_det_fst, lat_opts, NULL))
|
||||
throw std::runtime_error("could not determinize");
|
||||
|
||||
std::cout << "Compact version of determinized FST is:\n";
|
||||
{
|
||||
FstPrinter<CompactArc> fstprinter(compact_det_fst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
assert(RandEquivalent(compact_det_fst, compact_fst, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length, max*/));
|
||||
} catch (...) {
|
||||
std::cout << "Failed to lattice-determinize this FST (probably not determinizable)\n";
|
||||
}
|
||||
delete fst;
|
||||
}
|
||||
}
|
||||
|
||||
// test that determinization proceeds correctly on acyclic FSTs
|
||||
// (guaranteed determinizable in this sense).
|
||||
template<class Arc> void TestDeterminizeLattice2() {
|
||||
RandFstOptions opts;
|
||||
opts.acyclic = true;
|
||||
for(int i = 0; i < 100; i++) {
|
||||
VectorFst<Arc> *fst = RandFst<Arc>(opts);
|
||||
std::cout << "FST before lattice-determinizing is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
VectorFst<Arc> ofst;
|
||||
DeterminizeLattice<TropicalWeight, int32>(*fst, &ofst);
|
||||
std::cout << "FST after lattice-determinizing is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(ofst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
delete fst;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
int main() {
|
||||
using namespace fst;
|
||||
TestLatticeStringRepository();
|
||||
TestDeterminizeLattice<StdArc>();
|
||||
TestDeterminizeLattice2<StdArc>();
|
||||
std::cout << "Tests succeeded\n";
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
// fstext/determinize-lattice.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_DETERMINIZE_LATTICE_H_
|
||||
#define KALDI_FSTEXT_DETERMINIZE_LATTICE_H_
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
#include "fstext/lattice-weight.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
/// \addtogroup fst_extensions
|
||||
/// @{
|
||||
|
||||
|
||||
// For example of usage, see test-determinize-lattice.cc
|
||||
|
||||
/*
|
||||
DeterminizeLattice implements a special form of determinization
|
||||
with epsilon removal, optimized for a phase of lattice generation.
|
||||
Its input is an FST with weight-type BaseWeightType (usually a pair of floats,
|
||||
with a lexicographical type of order, such as LatticeWeightTpl<float>).
|
||||
Typically this would be a state-level lattice, with input symbols equal to
|
||||
words, and output-symbols equal to p.d.f's (so like the inverse of HCLG). Imagine representing this as an
|
||||
acceptor of type CompactLatticeWeightTpl<float>, in which the input/output
|
||||
symbols are words, and the weights contain the original weights together with
|
||||
strings (with zero or one symbol in them) containing the original output labels
|
||||
(the p.d.f.'s). We determinize this using acceptor determinization with
|
||||
epsilon removal. Remember (from lattice-weight.h) that
|
||||
CompactLatticeWeightTpl has a special kind of semiring where we always take
|
||||
the string corresponding to the best cost (of type BaseWeightType), and
|
||||
discard the other. This corresponds to taking the best output-label sequence
|
||||
(of p.d.f.'s) for each input-label sequence (of words). We couldn't use the
|
||||
Gallic weight for this, or it would die as soon as it detected that the input
|
||||
FST was non-functional. In our case, any acyclic FST (and many cyclic ones)
|
||||
can be determinized.
|
||||
We assume that there is a function
|
||||
Compare(const BaseWeightType &a, const BaseWeightType &b)
|
||||
that returns (-1, 0, 1) according to whether (a < b, a == b, a > b) in the
|
||||
total order on the BaseWeightType... this information should be the
|
||||
same as NaturalLess would give, but it's more efficient to do it this way.
|
||||
You can define this for things like TropicalWeight if you need to instantiate
|
||||
this class for that weight type.
|
||||
|
||||
We implement this determinization in a special way to make it efficient for
|
||||
the types of FSTs that we will apply it to. One issue is that if we
|
||||
explicitly represent the strings (in CompactLatticeWeightTpl) as vectors of
|
||||
type vector<IntType>, the algorithm takes time quadratic in the length of
|
||||
words (in states), because propagating each arc involves copying a whole
|
||||
vector (of integers representing p.d.f.'s). Instead we use a hash structure
|
||||
where each string is a pointer (Entry*), and uses a hash from (Entry*,
|
||||
IntType), to the successor string (and a way to get the latest IntType and the
|
||||
ancestor Entry*). [this is the class LatticeStringRepository].
|
||||
|
||||
Another issue is that rather than representing a determinized-state as a
|
||||
collection of (state, weight), we represent it in a couple of reduced forms.
|
||||
Suppose a determinized-state is a collection of (state, weight) pairs; call
|
||||
this the "canonical representation". Note: these collections are always
|
||||
normalized to remove any common weight and string part. Define end-states as
|
||||
the subset of states that have an arc out of them with a label on, or are
|
||||
final. If we represent a determinized-state a the set of just its (end-state,
|
||||
weight) pairs, this will be a valid and more compact representation, and will
|
||||
lead to a smaller set of determinized states (like early minimization). Call
|
||||
this collection of (end-state, weight) pairs the "minimal representation". As
|
||||
a mechanism to reduce compute, we can also consider another representation.
|
||||
In the determinization algorithm, we start off with a set of (begin-state,
|
||||
weight) pairs (where the "begin-states" are initial or have a label on the
|
||||
transition into them), and the "canonical representation" consists of the
|
||||
epsilon-closure of this set (i.e. follow epsilons). Call this set of
|
||||
(begin-state, weight) pairs, appropriately normalized, the "initial
|
||||
representation". If two initial representations are the same, the "canonical
|
||||
representation" and hence the "minimal representation" will be the same. We
|
||||
can use this to reduce compute. Note that if two initial representations are
|
||||
different, this does not preclude the other representations from being the same.
|
||||
|
||||
*/
|
||||
|
||||
struct DeterminizeLatticeOptions {
|
||||
float delta; // A small offset used to measure equality of weights.
|
||||
int max_mem; // If >0, determinization will fail and return false
|
||||
// when the algorithm's (approximate) memory consumption crosses this threshold.
|
||||
int max_loop; // If >0, can be used to detect non-determinizable input
|
||||
// (a case that wouldn't be caught by max_mem).
|
||||
DeterminizeLatticeOptions(): delta(kDelta),
|
||||
max_mem(-1),
|
||||
max_loop(-1) { }
|
||||
};
|
||||
|
||||
/**
|
||||
This function implements the normal version of DeterminizeLattice, in which
|
||||
the output strings are represented using sequences of arcs, where all but
|
||||
the first one has an epsilon on the input side. The debug_ptr argument is
|
||||
an optional pointer to a bool that, if it becomes true while the algorithm
|
||||
is executing, the algorithm will print a traceback and terminate (used in
|
||||
fstdeterminizestar.cc debug non-terminating determinization). More
|
||||
efficient if ifst is arc-sorted on input label. If the number of arcs gets
|
||||
more than max_states, it will throw std::runtime_error (otherwise this code
|
||||
does not use exceptions). This is mainly useful for debug. */
|
||||
template<class Weight, class IntType>
|
||||
bool DeterminizeLattice(
|
||||
const Fst<ArcTpl<Weight> > &ifst,
|
||||
MutableFst<ArcTpl<Weight> > *ofst,
|
||||
DeterminizeLatticeOptions opts = DeterminizeLatticeOptions(),
|
||||
bool *debug_ptr = NULL);
|
||||
|
||||
|
||||
/* This is a version of DeterminizeLattice with a slightly more "natural" output format,
|
||||
where the output sequences are encoded using the CompactLatticeArcTpl template
|
||||
(i.e. the sequences of output symbols are represented directly as strings)
|
||||
More efficient if ifst is arc-sorted on input label.
|
||||
If the #arcs gets more than max_arcs, it will throw std::runtime_error (otherwise
|
||||
this code does not use exceptions). This is mainly useful for debug.
|
||||
*/
|
||||
template<class Weight, class IntType>
|
||||
bool DeterminizeLattice(
|
||||
const Fst<ArcTpl<Weight> >&ifst,
|
||||
MutableFst<ArcTpl<CompactLatticeWeightTpl<Weight, IntType> > > *ofst,
|
||||
DeterminizeLatticeOptions opts = DeterminizeLatticeOptions(),
|
||||
bool *debug_ptr = NULL);
|
||||
|
||||
|
||||
|
||||
|
||||
/// @} end "addtogroup fst_extensions"
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
#include "fstext/determinize-lattice-inl.h"
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,508 @@
|
||||
// fstext/determinize-star-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2015 Hainan Xu
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "base/kaldi-math.h"
|
||||
#include "fstext/pre-determinize.h"
|
||||
#include "fstext/determinize-star.h"
|
||||
#include "fstext/trivial-factor-weight.h"
|
||||
#include "fstext/fst-test-utils.h"
|
||||
|
||||
|
||||
namespace fst
|
||||
{
|
||||
|
||||
// test that determinization proceeds correctly on general
|
||||
// FSTs (not guaranteed determinzable, but we use the
|
||||
// max-states option to stop it getting out of control).
|
||||
template<class Arc> void TestDeterminizeGeneral() {
|
||||
int max_states = 100; // don't allow more det-states than this.
|
||||
for(int i = 0; i < 100; i++) {
|
||||
VectorFst<Arc> *fst = RandFst<Arc>();
|
||||
std::cout << "FST before determinizing is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
VectorFst<Arc> ofst;
|
||||
try {
|
||||
DeterminizeStar<Fst<Arc> >(*fst, &ofst, kDelta, NULL, max_states);
|
||||
std::cout << "FST after determinizing is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(ofst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
assert(RandEquivalent(*fst, ofst, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length, max*/));
|
||||
} catch (...) {
|
||||
std::cout << "Failed to determinize *this FST (probably not determinizable)\n";
|
||||
}
|
||||
delete fst;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Don't instantiate with log semiring, as RandEquivalent may fail.
|
||||
template<class Arc> void TestDeterminize() {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> *fst = new VectorFst<Arc>();
|
||||
int n_syms = 2 + kaldi::Rand() % 5, n_states = 3 + kaldi::Rand() % 10, n_arcs = 5 + kaldi::Rand() % 30, n_final = 1 + kaldi::Rand()%3; // Up to 2 unique symbols.
|
||||
std::cout << "Testing pre-determinize with "<<n_syms<<" symbols, "<<n_states<<" states and "<<n_arcs<<" arcs and "<<n_final<<" final states.\n";
|
||||
SymbolTable *sptr = NULL;
|
||||
|
||||
std::vector<Label> all_syms; // including epsilon.
|
||||
// Put symbols in the symbol table from 1..n_syms-1.
|
||||
for (size_t i = 0;i < (size_t)n_syms;i++)
|
||||
all_syms.push_back(i);
|
||||
|
||||
// Create states.
|
||||
std::vector<StateId> all_states;
|
||||
for (size_t i = 0;i < (size_t)n_states;i++) {
|
||||
StateId this_state = fst->AddState();
|
||||
if (i == 0) fst->SetStart(i);
|
||||
all_states.push_back(this_state);
|
||||
}
|
||||
// Set final states.
|
||||
for (size_t j = 0;j < (size_t)n_final;j++) {
|
||||
StateId id = all_states[kaldi::Rand() % n_states];
|
||||
Weight weight = (Weight)(0.33*(kaldi::Rand() % 5) );
|
||||
printf("calling SetFinal with %d and %f\n", id, weight.Value());
|
||||
fst->SetFinal(id, weight);
|
||||
}
|
||||
// Create arcs.
|
||||
for (size_t i = 0;i < (size_t)n_arcs;i++) {
|
||||
Arc a;
|
||||
a.nextstate = all_states[kaldi::Rand() % n_states];
|
||||
a.ilabel = all_syms[kaldi::Rand() % n_syms];
|
||||
a.olabel = all_syms[kaldi::Rand() % n_syms]; // same input+output vocab.
|
||||
a.weight = (Weight) (0.33*(kaldi::Rand() % 2));
|
||||
StateId start_state = all_states[kaldi::Rand() % n_states];
|
||||
fst->AddArc(start_state, a);
|
||||
}
|
||||
|
||||
std::cout <<" printing before trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
// Trim resulting FST.
|
||||
Connect(fst);
|
||||
|
||||
std::cout <<" printing after trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
VectorFst<Arc> *fst_copy_orig = new VectorFst<Arc>(*fst);
|
||||
|
||||
std::vector<Label> extra_syms;
|
||||
if (fst->Start() != kNoStateId) { // "Connect" did not make it empty....
|
||||
PreDeterminize(fst, 1000, &extra_syms);
|
||||
}
|
||||
|
||||
std::cout <<" printing after predeterminization\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
{ // Remove epsilon. All default args.
|
||||
bool connect = true;
|
||||
Weight weight_threshold = Weight::Zero();
|
||||
int64 nstate = -1; // Relates to pruning.
|
||||
double delta = kDelta; // I think a small weight value. Relates to some kind of pruning,
|
||||
// I guess. But with no epsilon cycles, probably doensn't matter.
|
||||
RmEpsilon(fst, connect, weight_threshold, nstate, delta);
|
||||
}
|
||||
|
||||
std::cout <<" printing after epsilon removal\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
VectorFst<Arc> ofst_orig;
|
||||
VectorFst<Arc> ofst_star;
|
||||
|
||||
{
|
||||
printf("Determinizing with baseline\n");
|
||||
DeterminizeOptions<Arc> opts; // Default options.
|
||||
Determinize(*fst, &ofst_orig, opts);
|
||||
}
|
||||
|
||||
{
|
||||
printf("Determinizing with DeterminizeStar\n");
|
||||
DeterminizeStar(*fst, &ofst_star);
|
||||
}
|
||||
|
||||
{
|
||||
std::cout <<" printing after determinization [baseline]\n";
|
||||
FstPrinter<Arc> fstprinter(ofst_orig, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
assert(ofst_orig.Properties(kIDeterministic, true) == kIDeterministic);
|
||||
}
|
||||
|
||||
{
|
||||
std::cout <<" printing after determinization [star]\n";
|
||||
FstPrinter<Arc> fstprinter(ofst_star, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
assert(ofst_star.Properties(kIDeterministic, true) == kIDeterministic);
|
||||
}
|
||||
|
||||
assert(RandEquivalent(ofst_orig, ofst_star, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
int64 num_removed = DeleteISymbols(&ofst_star, extra_syms);
|
||||
std::cout <<" printing after removing "<<num_removed<<" instances of extra symbols\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(ofst_star, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
std::cout <<" Checking equivalent to original FST.\n";
|
||||
// giving Rand() as a seed stops the random number generator from always being reset to
|
||||
// the same point each time, while maintaining determinism of the test.
|
||||
assert(RandEquivalent(ofst_star, *fst_copy_orig, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
delete fst;
|
||||
delete fst_copy_orig;
|
||||
}
|
||||
|
||||
// Don't call this-- the test will fail due to the FST being non-functional.
|
||||
template<class Arc> void TestDeterminize2() {
|
||||
for(int i = 0; i < 10; i++) {
|
||||
RandFstOptions opts;
|
||||
opts.acyclic = true;
|
||||
VectorFst<Arc> *ifst = RandFst<Arc>(opts);
|
||||
VectorFst<Arc> ofst;
|
||||
Determinize(*ifst, &ofst);
|
||||
assert(RandEquivalent(*ifst, ofst, 5, 0.01, kaldi::Rand(), 100));
|
||||
delete ifst;
|
||||
}
|
||||
}
|
||||
|
||||
template<class Arc> void TestPush() {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> *fst = new VectorFst<Arc>();
|
||||
int n_syms = 2 + kaldi::Rand() % 5, n_states = 3 + kaldi::Rand() % 10, n_arcs = 5 + kaldi::Rand() % 30, n_final = 1 + kaldi::Rand()%3; // Up to 2 unique symbols.
|
||||
std::cout << "Testing pre-determinize with "<<n_syms<<" symbols, "<<n_states<<" states and "<<n_arcs<<" arcs and "<<n_final<<" final states.\n";
|
||||
SymbolTable *sptr = NULL;
|
||||
|
||||
std::vector<Label> all_syms; // including epsilon.
|
||||
// Put symbols in the symbol table from 1..n_syms-1.
|
||||
for (size_t i = 0;i < (size_t)n_syms;i++)
|
||||
all_syms.push_back(i);
|
||||
|
||||
// Create states.
|
||||
std::vector<StateId> all_states;
|
||||
for (size_t i = 0;i < (size_t)n_states;i++) {
|
||||
StateId this_state = fst->AddState();
|
||||
if (i == 0) fst->SetStart(i);
|
||||
all_states.push_back(this_state);
|
||||
}
|
||||
// Set final states.
|
||||
for (size_t j = 0;j < (size_t)n_final;j++) {
|
||||
StateId id = all_states[kaldi::Rand() % n_states];
|
||||
Weight weight = (Weight)(0.33*(kaldi::Rand() % 5) );
|
||||
printf("calling SetFinal with %d and %f\n", id, weight.Value());
|
||||
fst->SetFinal(id, weight);
|
||||
}
|
||||
// Create arcs.
|
||||
for (size_t i = 0;i < (size_t)n_arcs;i++) {
|
||||
Arc a;
|
||||
a.nextstate = all_states[kaldi::Rand() % n_states];
|
||||
a.ilabel = all_syms[kaldi::Rand() % n_syms];
|
||||
a.olabel = all_syms[kaldi::Rand() % n_syms]; // same input+output vocab.
|
||||
a.weight = (Weight) (0.33*(kaldi::Rand() % 2));
|
||||
StateId start_state = all_states[kaldi::Rand() % n_states];
|
||||
fst->AddArc(start_state, a);
|
||||
}
|
||||
|
||||
std::cout <<" printing before trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
// Trim resulting FST.
|
||||
Connect(fst);
|
||||
|
||||
std::cout <<" printing after trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
VectorFst<Arc> *fst_copy_orig = new VectorFst<Arc>(*fst);
|
||||
|
||||
std::vector<Label> extra_syms;
|
||||
if (fst->Start() != kNoStateId) { // "Connect" did not make it empty....
|
||||
PreDeterminize(fst, 1000, &extra_syms);
|
||||
}
|
||||
|
||||
VectorFst<Arc> fst_pushed;
|
||||
std::cout << "Pushing FST\n";
|
||||
Push<Arc, REWEIGHT_TO_INITIAL>(*fst, &fst_pushed, kPushWeights|kPushLabels, kDelta);
|
||||
|
||||
std::cout <<" printing after pushing\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(fst_pushed, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
assert(RandEquivalent(*fst, fst_pushed, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
delete fst;
|
||||
delete fst_copy_orig;
|
||||
}
|
||||
|
||||
// Don't instantiate with log semiring, as RandEquivalent may fail.
|
||||
template<class Arc> void TestMinimize() {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> *fst = new VectorFst<Arc>();
|
||||
int n_syms = 2 + kaldi::Rand() % 5, n_states = 3 + kaldi::Rand() % 10, n_arcs = 5 + kaldi::Rand() % 30, n_final = 1 + kaldi::Rand()%3; // Up to 2 unique symbols.
|
||||
std::cout << "Testing pre-determinize with "<<n_syms<<" symbols, "<<n_states<<" states and "<<n_arcs<<" arcs and "<<n_final<<" final states.\n";
|
||||
SymbolTable *sptr =NULL;
|
||||
|
||||
std::vector<Label> all_syms; // including epsilon.
|
||||
// Put symbols in the symbol table from 1..n_syms-1.
|
||||
for (size_t i = 0;i < (size_t)n_syms;i++)
|
||||
all_syms.push_back(i);
|
||||
|
||||
// Create states.
|
||||
std::vector<StateId> all_states;
|
||||
for (size_t i = 0;i < (size_t)n_states;i++) {
|
||||
StateId this_state = fst->AddState();
|
||||
if (i == 0) fst->SetStart(i);
|
||||
all_states.push_back(this_state);
|
||||
}
|
||||
// Set final states.
|
||||
for (size_t j = 0;j < (size_t)n_final;j++) {
|
||||
StateId id = all_states[kaldi::Rand() % n_states];
|
||||
Weight weight = (Weight)(0.33*(kaldi::Rand() % 5) );
|
||||
printf("calling SetFinal with %d and %f\n", id, weight.Value());
|
||||
fst->SetFinal(id, weight);
|
||||
}
|
||||
// Create arcs.
|
||||
for (size_t i = 0;i < (size_t)n_arcs;i++) {
|
||||
Arc a;
|
||||
a.nextstate = all_states[kaldi::Rand() % n_states];
|
||||
a.ilabel = all_syms[kaldi::Rand() % n_syms];
|
||||
a.olabel = all_syms[kaldi::Rand() % n_syms]; // same input+output vocab.
|
||||
a.weight = (Weight) (0.33*(kaldi::Rand() % 2));
|
||||
StateId start_state = all_states[kaldi::Rand() % n_states];
|
||||
fst->AddArc(start_state, a);
|
||||
}
|
||||
|
||||
std::cout <<" printing before trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
// Trim resulting FST.
|
||||
Connect(fst);
|
||||
|
||||
std::cout <<" printing after trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
VectorFst<Arc> *fst_copy_orig = new VectorFst<Arc>(*fst);
|
||||
|
||||
std::vector<Label> extra_syms;
|
||||
if (fst->Start() != kNoStateId) { // "Connect" did not make it empty....
|
||||
PreDeterminize(fst, 1000, &extra_syms);
|
||||
}
|
||||
|
||||
std::cout <<" printing after predeterminization\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
{ // Remove epsilon. All default args.
|
||||
bool connect = true;
|
||||
Weight weight_threshold = Weight::Zero();
|
||||
int64 nstate = -1; // Relates to pruning.
|
||||
double delta = kDelta; // I think a small weight value. Relates to some kind of pruning,
|
||||
// I guess. But with no epsilon cycles, probably doensn't matter.
|
||||
RmEpsilon(fst, connect, weight_threshold, nstate, delta);
|
||||
}
|
||||
|
||||
std::cout <<" printing after epsilon removal\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
VectorFst<Arc> ofst_orig;
|
||||
VectorFst<Arc> ofst_star;
|
||||
|
||||
{
|
||||
printf("Determinizing with baseline\n");
|
||||
DeterminizeOptions<Arc> opts; // Default options.
|
||||
Determinize(*fst, &ofst_orig, opts);
|
||||
}
|
||||
{
|
||||
std::cout <<" printing after determinization [baseline]\n";
|
||||
FstPrinter<Arc> fstprinter(ofst_orig, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
{
|
||||
printf("Determinizing with DeterminizeStar to Gallic semiring\n");
|
||||
VectorFst<GallicArc<Arc> > gallic_fst;
|
||||
|
||||
DeterminizeStar(*fst, &gallic_fst);
|
||||
{
|
||||
std::cout <<" printing after determinization by DeterminizeStar [in gallic]\n";
|
||||
FstPrinter<GallicArc< Arc> > fstprinter(gallic_fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
printf("Pushing weights\n");
|
||||
Push(&gallic_fst, REWEIGHT_TO_INITIAL, kDelta);
|
||||
|
||||
{
|
||||
std::cout <<" printing after pushing weights [in gallic]\n";
|
||||
FstPrinter<GallicArc< Arc> > fstprinter(gallic_fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
printf("Minimizing [in Gallic]\n");
|
||||
Minimize(&gallic_fst);
|
||||
{
|
||||
std::cout <<" printing after minimization [in gallic]\n";
|
||||
FstPrinter<GallicArc< Arc> > fstprinter(gallic_fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
printf("Converting gallic back to regular [my approach]\n");
|
||||
TrivialFactorWeightFst< GallicArc<Arc, GALLIC_LEFT>, GallicFactor<typename Arc::Label,
|
||||
typename Arc::Weight, GALLIC_LEFT> > fwfst(gallic_fst);
|
||||
{
|
||||
std::cout <<" printing factor-weight FST\n";
|
||||
FstPrinter<GallicArc< Arc> > fstprinter(fwfst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
Map(fwfst, &ofst_star, FromGallicMapper<Arc, GALLIC_LEFT>());
|
||||
|
||||
{
|
||||
std::cout <<" printing after converting back to regular FST\n";
|
||||
FstPrinter<Arc> fstprinter(ofst_star, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
assert(RandEquivalent(ofst_orig, ofst_star, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
|
||||
int64 num_removed = DeleteISymbols(&ofst_star, extra_syms);
|
||||
std::cout <<" printing after removing "<<num_removed<<" instances of extra symbols\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(ofst_star, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
std::cout <<" Checking equivalent to original FST.\n";
|
||||
// giving Rand() as a seed stops the random number generator from always being reset to
|
||||
// the same point each time, while maintaining determinism of the test.
|
||||
assert(RandEquivalent(ofst_star, *fst_copy_orig, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
delete fst;
|
||||
delete fst_copy_orig;
|
||||
}
|
||||
|
||||
|
||||
template<class Arc, class inttype> void TestStringRepository() {
|
||||
typedef typename Arc::Label Label;
|
||||
|
||||
StringRepository<Label, inttype> sr;
|
||||
|
||||
int N = 100;
|
||||
if (sizeof(inttype) == 1) N = 64;
|
||||
std::vector<std::vector<Label> > strings(N);
|
||||
std::vector<inttype> ids(N);
|
||||
|
||||
for (int i = 0;i < N;i++) {
|
||||
size_t len = kaldi::Rand() % 4;
|
||||
std::vector<Label> vec;
|
||||
for (size_t j = 0;j < len;j++) vec.push_back( (kaldi::Rand()%10) + 150*(kaldi::Rand()%2)); // make it have reasonable range.
|
||||
if (i < 500 && vec.size() == 0) ids[i] = sr.IdOfEmpty();
|
||||
else if (i < 500 && vec.size() == 1) ids[i] = sr.IdOfLabel(vec[0]);
|
||||
else ids[i] = sr.IdOfSeq(vec);
|
||||
|
||||
strings[i] = vec;
|
||||
}
|
||||
|
||||
for (int i = 0;i < N;i++) {
|
||||
std::vector<Label> tmpv;
|
||||
tmpv.push_back(10); // just put in garbage.
|
||||
sr.SeqOfId(ids[i], &tmpv);
|
||||
assert(tmpv == strings[i]);
|
||||
assert(sr.IdOfSeq(strings[i]) == ids[i]);
|
||||
if (strings[i].size() == 0) assert(ids[i] == sr.IdOfEmpty());
|
||||
if (strings[i].size() == 1) assert(ids[i] == sr.IdOfLabel(strings[i][0]));
|
||||
|
||||
if (sizeof(inttype) != 1) {
|
||||
size_t prefix_len = kaldi::Rand() % (strings[i].size() + 1);
|
||||
inttype s2 = sr.RemovePrefix(ids[i], prefix_len);
|
||||
std::vector<Label> vec2;
|
||||
sr.SeqOfId(s2, &vec2);
|
||||
for (size_t j = 0;j < strings[i].size()-prefix_len;j++) {
|
||||
assert(vec2[j] == strings[i][j+prefix_len]);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
|
||||
int main() {
|
||||
for (int i = 0;i < 3;i++) { // We would need more iterations to check
|
||||
// this properly.
|
||||
fst::TestStringRepository<fst::StdArc, int>();
|
||||
fst::TestStringRepository<fst::StdArc, unsigned int>();
|
||||
// Not for use with char, but this helps reveal some kinds of bugs.
|
||||
fst::TestStringRepository<fst::StdArc, unsigned char>();
|
||||
fst::TestStringRepository<fst::StdArc, char>();
|
||||
fst::TestDeterminizeGeneral<fst::StdArc>();
|
||||
fst::TestDeterminize<fst::StdArc>();
|
||||
// fst::TestDeterminize2<fst::StdArc>();
|
||||
fst::TestPush<fst::StdArc>();
|
||||
fst::TestMinimize<fst::StdArc>();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
// fstext/determinize-star.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2014 Guoguo Chen
|
||||
// 2015 Hainan Xu
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_DETERMINIZE_STAR_H_
|
||||
#define KALDI_FSTEXT_DETERMINIZE_STAR_H_
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
#include <stdexcept> // this algorithm uses exceptions
|
||||
|
||||
namespace fst {
|
||||
|
||||
/// \addtogroup fst_extensions
|
||||
/// @{
|
||||
|
||||
|
||||
// For example of usage, see test-determinize-star.cc
|
||||
|
||||
/*
|
||||
DeterminizeStar implements determinization with epsilon removal, which we
|
||||
distinguish with a star.
|
||||
|
||||
We define a determinized* FST as one in which no state has more than one
|
||||
transition with the same input-label. Epsilon input labels are not allowed
|
||||
except starting from states that have exactly one arc exiting them (and are
|
||||
not final). [In the normal definition of determinized, epsilon-input labels
|
||||
are not allowed at all, whereas in Mohri's definition, epsilons are treated
|
||||
as ordinary symbols]. The determinized* definition is intended to simulate
|
||||
the effect of allowing strings of output symbols at each state.
|
||||
|
||||
The algorithm implemented here takes an Fst<Arc>, and a pointer to a
|
||||
MutableFst<Arc> where it puts its output. The weight type is assumed to be a
|
||||
float-weight. It does epsilon removal and determinization.
|
||||
This algorithm may fail if the input has epsilon cycles under
|
||||
certain circumstances (i.e. the semiring is non-idempotent, e.g. the log
|
||||
semiring, or there are negative cost epsilon cycles).
|
||||
|
||||
This implementation is much less fancy than the one in fst/determinize.h, and
|
||||
does not have an "on-demand" version.
|
||||
|
||||
The algorithm is a fairly normal determinization algorithm. We keep in
|
||||
memory the subsets of states, together with their leftover strings and their
|
||||
weights. The only difference is we detect input epsilon transitions and
|
||||
treat them "specially".
|
||||
*/
|
||||
|
||||
|
||||
// This algorithm will be slightly faster if you sort the input fst on input label.
|
||||
|
||||
/**
|
||||
This function implements the normal version of DeterminizeStar, in which the
|
||||
output strings are represented using sequences of arcs, where all but the
|
||||
first one has an epsilon on the input side. The debug_ptr argument is an
|
||||
optional pointer to a bool that, if it becomes true while the algorithm is
|
||||
executing, the algorithm will print a traceback and terminate (used in
|
||||
fstdeterminizestar.cc debug non-terminating determinization).
|
||||
If max_states is positive, it will stop determinization and throw an
|
||||
exception as soon as the max-states is reached. This can be useful in test.
|
||||
If allow_partial is true, the algorithm will output partial results when the
|
||||
specified max_states is reached (when larger than zero), instead of throwing
|
||||
out an error.
|
||||
|
||||
Caution, the return status is un-intuitive: this function will return false if
|
||||
determinization completed normally, and true if it was stopped early by
|
||||
reaching the 'max-states' limit, and a partial FST was generated.
|
||||
*/
|
||||
template<class F>
|
||||
bool DeterminizeStar(F &ifst, MutableFst<typename F::Arc> *ofst,
|
||||
float delta = kDelta,
|
||||
bool *debug_ptr = NULL,
|
||||
int max_states = -1,
|
||||
bool allow_partial = false);
|
||||
|
||||
|
||||
|
||||
/* This is a version of DeterminizeStar with a slightly more "natural" output format,
|
||||
where the output sequences are encoded using the GallicArc (i.e. the output symbols
|
||||
are strings.
|
||||
If max_states is positive, it will stop determinization and throw an
|
||||
exception as soon as the max-states is reached. This can be useful in test.
|
||||
If allow_partial is true, the algorithm will output partial results when the
|
||||
specified max_states is reached (when larger than zero), instead of throwing
|
||||
out an error.
|
||||
|
||||
Caution, the return status is un-intuitive: this function will return false if
|
||||
determinization completed normally, and true if it was stopped early by
|
||||
reaching the 'max-states' limit, and a partial FST was generated.
|
||||
*/
|
||||
template<class F>
|
||||
bool DeterminizeStar(F &ifst, MutableFst<GallicArc<typename F::Arc> > *ofst,
|
||||
float delta = kDelta, bool *debug_ptr = NULL,
|
||||
int max_states = -1,
|
||||
bool allow_partial = false);
|
||||
|
||||
|
||||
/// @} end "addtogroup fst_extensions"
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
#include "fstext/determinize-star-inl.h"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,126 @@
|
||||
// fstext/epsilon-property-inl.h
|
||||
|
||||
// Copyright 2014 Johns Hopkins University (Author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_EPSILON_PROPERTY_INL_H_
|
||||
#define KALDI_FSTEXT_EPSILON_PROPERTY_INL_H_
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
|
||||
template<class Arc>
|
||||
void ComputeStateInfo(const VectorFst<Arc> &fst,
|
||||
std::vector<char> *epsilon_info) {
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef VectorFst<Arc> Fst;
|
||||
epsilon_info->clear();
|
||||
epsilon_info->resize(fst.NumStates(), static_cast<char>(0));
|
||||
for (StateId s = 0; s < fst.NumStates(); s++) {
|
||||
for (ArcIterator<Fst> aiter(fst, s); !aiter.Done(); aiter.Next()) {
|
||||
const Arc &arc = aiter.Value();
|
||||
if (arc.ilabel == 0 && arc.olabel == 0) {
|
||||
(*epsilon_info)[arc.nextstate] |= static_cast<char>(kStateHasEpsilonArcsEntering);
|
||||
(*epsilon_info)[s] |= static_cast<char>(kStateHasEpsilonArcsLeaving);
|
||||
} else {
|
||||
(*epsilon_info)[arc.nextstate] |= static_cast<char>(kStateHasNonEpsilonArcsEntering);
|
||||
(*epsilon_info)[s] |= static_cast<char>(kStateHasNonEpsilonArcsLeaving);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
void EnsureEpsilonProperty(VectorFst<Arc> *fst) {
|
||||
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef VectorFst<Arc> Fst;
|
||||
std::vector<char> epsilon_info;
|
||||
ComputeStateInfo(*fst, &epsilon_info);
|
||||
|
||||
|
||||
StateId num_states_old = fst->NumStates();
|
||||
StateId non_coaccessible_state = fst->AddState();
|
||||
|
||||
/// new_state_vec is for those states that have both epsilon and
|
||||
/// non-epsilon arcs entering. For these states, we'll create a new
|
||||
/// state for the non-epsilon arcs to enter and put it in this array,
|
||||
/// and we'll put an epsilon transition from the new state to the old state.
|
||||
std::vector<StateId> new_state_vec(num_states_old, kNoStateId);
|
||||
for (StateId s = 0; s < num_states_old; s++) {
|
||||
if ((epsilon_info[s] & kStateHasEpsilonArcsEntering) != 0 &&
|
||||
(epsilon_info[s] & kStateHasNonEpsilonArcsEntering) != 0) {
|
||||
assert(s != fst->Start()); // a type of cyclic FST we can't handle
|
||||
// easily.
|
||||
StateId new_state = fst->AddState();
|
||||
new_state_vec[s] = new_state;
|
||||
fst->AddArc(new_state, Arc(0, 0, Weight::One(), s));
|
||||
}
|
||||
}
|
||||
|
||||
/// First modify arcs to point to states in new_state_vec when
|
||||
/// necessary.
|
||||
for (StateId s = 0; s < num_states_old; s++) {
|
||||
for (MutableArcIterator<Fst> aiter(fst, s);
|
||||
!aiter.Done(); aiter.Next()) {
|
||||
Arc arc = aiter.Value();
|
||||
if (arc.ilabel != 0 || arc.olabel != 0) { // non-epsilon arc
|
||||
StateId replacement_state;
|
||||
if (arc.nextstate >= 0 && arc.nextstate < num_states_old &&
|
||||
(replacement_state = new_state_vec[arc.nextstate]) !=
|
||||
kNoStateId) {
|
||||
arc.nextstate = replacement_state;
|
||||
aiter.SetValue(arc);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Now handle the situation where states have both epsilon and non-epsilon
|
||||
/// arcs leaving.
|
||||
for (StateId s = 0; s < num_states_old; s++) {
|
||||
if ((epsilon_info[s] & kStateHasEpsilonArcsLeaving) != 0 &&
|
||||
(epsilon_info[s] & kStateHasNonEpsilonArcsLeaving) != 0) {
|
||||
// state has non-epsilon and epsilon arcs leaving.
|
||||
// create a new state and move the non-epsilon arcs to leave
|
||||
// from there instead.
|
||||
StateId new_state = fst->AddState();
|
||||
for (MutableArcIterator<Fst> aiter(fst, s); !aiter.Done();
|
||||
aiter.Next()) {
|
||||
Arc arc = aiter.Value();
|
||||
if (arc.ilabel != 0 || arc.olabel != 0) { // non-epsilon arc.
|
||||
assert(arc.nextstate != s); // we don't handle cyclic FSTs.
|
||||
// move this arc to leave from the new state:
|
||||
fst->AddArc(new_state, arc);
|
||||
arc.nextstate = non_coaccessible_state;
|
||||
aiter.SetValue(arc); // invalidate the arc, Connect() will remove it.
|
||||
}
|
||||
}
|
||||
// Create an epsilon arc to the new state.
|
||||
fst->AddArc(s, Arc(0, 0, Weight::One(), new_state));
|
||||
}
|
||||
}
|
||||
Connect(fst); // Removes arcs to the non-coaccessible state.
|
||||
}
|
||||
|
||||
|
||||
|
||||
} // namespace fst.
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,58 @@
|
||||
// fstext/epsilon-property-test.cc
|
||||
|
||||
// Copyright 2014 Johns Hopkins University (Author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#include "fstext/rand-fst.h"
|
||||
#include "fstext/epsilon-property.h"
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
void TestEnsureEpsilonProperty() {
|
||||
|
||||
for (int32 i = 0; i < 10; i++) {
|
||||
RandFstOptions opts;
|
||||
opts.acyclic = true;
|
||||
VectorFst<LogArc> *fst = RandFst<LogArc>(opts);
|
||||
VectorFst<LogArc> fst2(*fst); // copy it...
|
||||
EnsureEpsilonProperty(&fst2);
|
||||
|
||||
std::vector<char> info;
|
||||
ComputeStateInfo(fst2, &info);
|
||||
for (size_t i = 0; i < info.size(); i++) {
|
||||
char c = info[i];
|
||||
assert(!((c & kStateHasEpsilonArcsEntering) != 0 &&
|
||||
(c & kStateHasNonEpsilonArcsEntering) != 0));
|
||||
assert(!((c & kStateHasEpsilonArcsLeaving) != 0 &&
|
||||
(c & kStateHasNonEpsilonArcsLeaving) != 0));
|
||||
}
|
||||
assert(RandEquivalent(fst2, *fst, 5, 0.01, kaldi::Rand(), 10));
|
||||
delete fst;
|
||||
}
|
||||
}
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
int main() {
|
||||
using namespace fst;
|
||||
for (int i = 0; i < 2; i++) {
|
||||
TestEnsureEpsilonProperty();
|
||||
}
|
||||
std::cout << "Test OK\n";
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// fstext/epsilon-property.h
|
||||
|
||||
// Copyright 2014 Johns Hopkins University (Author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_EPSILON_PROPERTY_H_
|
||||
#define KALDI_FSTEXT_EPSILON_PROPERTY_H_
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
|
||||
namespace fst {
|
||||
|
||||
enum {
|
||||
kStateHasEpsilonArcsEntering = 0x1,
|
||||
kStateHasNonEpsilonArcsEntering = 0x2,
|
||||
kStateHasEpsilonArcsLeaving = 0x4,
|
||||
kStateHasNonEpsilonArcsLeaving = 0x8
|
||||
}; // use 'char' for this enum.
|
||||
|
||||
/// This function will set epsilon_info to have size equal to the
|
||||
/// NumStates() of the FST, containing a logical-or of the enum
|
||||
/// values kStateHasEpsilonArcsEntering, kStateHasNonEpsilonArcsEntering,
|
||||
/// kStateHasEpsilonArcsLeaving, and kStateHasNonEpsilonArcsLeaving.
|
||||
/// The meaning should be obvious. Note: an epsilon arc is defined
|
||||
/// as an arc where ilabel == olabel == 0.
|
||||
template<class Arc>
|
||||
void ComputeStateInfo(const VectorFst<Arc> &fst,
|
||||
std::vector<char> *epsilon_info);
|
||||
|
||||
/// This function modifies the fst (while maintaining equivalence) in such a way
|
||||
/// that, after the modification, all states of the FST which have epsilon-arcs
|
||||
/// entering them, have no non-epsilon arcs entering them, and all states which
|
||||
/// have epsilon-arcs leaving them, have no non-epsilon arcs leaving them. It does
|
||||
/// this by creating extra states and adding extra epsilon transitions. An epsilon
|
||||
/// arc is defined as an arc where both the ilabel and the olabel are epsilons.
|
||||
/// This function may fail with KALDI_ASSERT for certain cyclic FSTs, but is safe
|
||||
/// in the acyclic case.
|
||||
template<class Arc>
|
||||
void EnsureEpsilonProperty(VectorFst<Arc> *fst);
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
|
||||
#include "fstext/epsilon-property-inl.h"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,284 @@
|
||||
// fstext/factor-inl.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_FACTOR_INL_H_
|
||||
#define KALDI_FSTEXT_FACTOR_INL_H_
|
||||
|
||||
#include "util/stl-utils.h"
|
||||
// Do not include this file directly. It is included by factor.h.
|
||||
|
||||
namespace fst {
|
||||
|
||||
// GetStateProperties takes in an FST and a number "max_state" which is the
|
||||
// highest numbered state in the FST (this could be fst.NumStates()-1 for an
|
||||
// ExpandedFst, or derived from some kind of traversal). It outputs a vector
|
||||
// numbered from 0..max_state, of type FstStateProperties which is a bitmask
|
||||
// with information about the states.
|
||||
|
||||
// GetStateProperties has not been tested directly (only implicitly via
|
||||
// testing Factor).
|
||||
template<class Arc>
|
||||
void GetStateProperties(const Fst<Arc> &fst,
|
||||
typename Arc::StateId max_state,
|
||||
std::vector<StatePropertiesType> *props) {
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
assert(props != NULL);
|
||||
props->clear();
|
||||
if (fst.Start() < 0) return; // Empty fst.
|
||||
props->resize(max_state+1, 0);
|
||||
assert(fst.Start() <= max_state);
|
||||
(*props)[fst.Start()] |= kStateInitial;
|
||||
for (StateId s = 0; s <= max_state; s++) {
|
||||
StatePropertiesType &s_info = (*props)[s];
|
||||
for (ArcIterator<Fst<Arc> > aiter(fst, s); !aiter.Done(); aiter.Next()) {
|
||||
const Arc &arc = aiter.Value();
|
||||
if (arc.ilabel != 0) s_info |= kStateIlabelsOut;
|
||||
if (arc.olabel != 0) s_info |= kStateOlabelsOut;
|
||||
StateId nexts = arc.nextstate;
|
||||
assert(nexts <= max_state); // or input was invalid.
|
||||
StatePropertiesType &nexts_info = (*props)[nexts];
|
||||
if (s_info&kStateArcsOut) s_info |= kStateMultipleArcsOut;
|
||||
s_info |= kStateArcsOut;
|
||||
if (nexts_info&kStateArcsIn) nexts_info |= kStateMultipleArcsIn;
|
||||
nexts_info |= kStateArcsIn;
|
||||
}
|
||||
if (fst.Final(s) != Weight::Zero()) s_info |= kStateFinal;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
template<class Arc, class I>
|
||||
void Factor(const Fst<Arc> &fst, MutableFst<Arc> *ofst,
|
||||
std::vector<std::vector<I> > *symbols_out) {
|
||||
KALDI_ASSERT_IS_INTEGER_TYPE(I);
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::Weight Weight;
|
||||
assert(symbols_out != NULL);
|
||||
ofst->DeleteStates();
|
||||
if (fst.Start() < 0) return; // empty FST.
|
||||
std::vector<StateId> order;
|
||||
DfsOrderVisitor<Arc> dfs_order_visitor(&order);
|
||||
DfsVisit(fst, &dfs_order_visitor);
|
||||
assert(order.size() > 0);
|
||||
StateId max_state = *(std::max_element(order.begin(), order.end()));
|
||||
std::vector<StatePropertiesType> state_properties;
|
||||
GetStateProperties(fst, max_state, &state_properties);
|
||||
|
||||
std::vector<bool> remove(max_state+1); // if true, will remove this state.
|
||||
|
||||
// Now identify states that will be removed (made the middle of a chain).
|
||||
// The basic rule is that if the FstStateProperties equals
|
||||
// (kStateArcsIn|kStateArcsOut) or (kStateArcsIn|kStateArcsOut|kStateIlabelsOut),
|
||||
// then it is in the middle of a chain. This eliminates state with
|
||||
// multiple input or output arcs, final states, and states with arcs out
|
||||
// that have olabels [we assume these are pushed to the left, so occur on the
|
||||
// 1st arc of a chain.
|
||||
|
||||
for (StateId i = 0; i <= max_state; i++)
|
||||
remove[i] = (state_properties[i] == (kStateArcsIn|kStateArcsOut)
|
||||
|| state_properties[i] == (kStateArcsIn|kStateArcsOut|kStateIlabelsOut));
|
||||
std::vector<StateId> state_mapping(max_state+1, kNoStateId);
|
||||
|
||||
typedef unordered_map<std::vector<I>, Label, kaldi::VectorHasher<I> > SymbolMapType;
|
||||
SymbolMapType symbol_mapping;
|
||||
Label symbol_counter = 0;
|
||||
{
|
||||
std::vector<I> eps;
|
||||
symbol_mapping[eps] = symbol_counter++;
|
||||
}
|
||||
std::vector<I> this_sym; // a temporary used inside the loop.
|
||||
for (size_t i = 0; i < order.size(); i++) {
|
||||
StateId state = order[i];
|
||||
if (!remove[state]) { // Process this state...
|
||||
StateId &new_state = state_mapping[state];
|
||||
if (new_state == kNoStateId) new_state = ofst->AddState();
|
||||
for (ArcIterator<Fst<Arc> > aiter(fst, state); !aiter.Done(); aiter.Next()) {
|
||||
Arc arc = aiter.Value();
|
||||
if (arc.ilabel == 0) this_sym.clear();
|
||||
else {
|
||||
this_sym.resize(1);
|
||||
this_sym[0] = arc.ilabel;
|
||||
}
|
||||
while (remove[arc.nextstate]) {
|
||||
ArcIterator<Fst<Arc> > aiter2(fst, arc.nextstate);
|
||||
assert(!aiter2.Done());
|
||||
const Arc &nextarc = aiter2.Value();
|
||||
arc.weight = Times(arc.weight, nextarc.weight);
|
||||
assert(nextarc.olabel == 0);
|
||||
if (nextarc.ilabel != 0) this_sym.push_back(nextarc.ilabel);
|
||||
assert(static_cast<Label>(static_cast<I>(nextarc.ilabel))
|
||||
== nextarc.ilabel); // check within integer range.
|
||||
arc.nextstate = nextarc.nextstate;
|
||||
}
|
||||
StateId &new_nextstate = state_mapping[arc.nextstate];
|
||||
if (new_nextstate == kNoStateId) new_nextstate = ofst->AddState();
|
||||
arc.nextstate = new_nextstate;
|
||||
if (symbol_mapping.count(this_sym) != 0) arc.ilabel = symbol_mapping[this_sym];
|
||||
else arc.ilabel = symbol_mapping[this_sym] = symbol_counter++;
|
||||
ofst->AddArc(new_state, arc);
|
||||
}
|
||||
if (fst.Final(state) != Weight::Zero())
|
||||
ofst->SetFinal(new_state, fst.Final(state));
|
||||
}
|
||||
}
|
||||
ofst->SetStart(state_mapping[fst.Start()]);
|
||||
|
||||
// Now output the symbol sequences.
|
||||
symbols_out->resize(symbol_counter);
|
||||
for (typename SymbolMapType::const_iterator iter = symbol_mapping.begin();
|
||||
iter != symbol_mapping.end(); ++iter) {
|
||||
(*symbols_out)[iter->second] = iter->first;
|
||||
}
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
void Factor(const Fst<Arc> &fst, MutableFst<Arc> *ofst1,
|
||||
MutableFst<Arc> *ofst2) {
|
||||
typedef typename Arc::Label Label;
|
||||
std::vector<std::vector<Label> > symbols;
|
||||
Factor(fst, ofst2, &symbols);
|
||||
CreateFactorFst(symbols, ofst1);
|
||||
}
|
||||
|
||||
template<class Arc, class I>
|
||||
void ExpandInputSequences(const std::vector<std::vector<I> > &sequences,
|
||||
MutableFst<Arc> *fst) {
|
||||
KALDI_ASSERT_IS_INTEGER_TYPE(I);
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::Weight Weight;
|
||||
fst->SetInputSymbols(NULL);
|
||||
size_t size = sequences.size();
|
||||
if (sequences.size() > 0) assert(sequences[0].size() == 0); // should be eps.
|
||||
StateId num_states_at_start = fst->NumStates();
|
||||
for (StateId s = 0; s < num_states_at_start; s++) {
|
||||
StateId num_arcs = fst->NumArcs(s);
|
||||
for (StateId aidx = 0; aidx < num_arcs; aidx++) {
|
||||
ArcIterator<MutableFst<Arc> > aiter(*fst, s);
|
||||
aiter.Seek(aidx);
|
||||
Arc arc = aiter.Value();
|
||||
|
||||
Label ilabel = arc.ilabel;
|
||||
Label dest_state = arc.nextstate;
|
||||
if (ilabel != 0) { // non-eps [nothing to do if eps]...
|
||||
assert(ilabel < static_cast<Label>(size));
|
||||
size_t len = sequences[ilabel].size();
|
||||
if (len <= 1) {
|
||||
if (len == 0) arc.ilabel = 0;
|
||||
else arc.ilabel = sequences[ilabel][0];
|
||||
MutableArcIterator<MutableFst<Arc> > mut_aiter(fst, s);
|
||||
mut_aiter.Seek(aidx);
|
||||
mut_aiter.SetValue(arc);
|
||||
} else { // len>=2. Must create new states...
|
||||
StateId curstate = -1; // keep compiler happy: this value never used.
|
||||
for (size_t n = 0; n < len; n++) { // adding/modifying "len" arcs.
|
||||
StateId nextstate;
|
||||
if (n < len-1) {
|
||||
nextstate = fst->AddState();
|
||||
assert(nextstate >= num_states_at_start);
|
||||
} else nextstate = dest_state; // going back to original arc's
|
||||
// destination.
|
||||
if (n == 0) {
|
||||
arc.ilabel = sequences[ilabel][0];
|
||||
arc.nextstate = nextstate;
|
||||
MutableArcIterator<MutableFst<Arc> > mut_aiter(fst, s);
|
||||
mut_aiter.Seek(aidx);
|
||||
mut_aiter.SetValue(arc);
|
||||
} else {
|
||||
arc.ilabel = sequences[ilabel][n];
|
||||
arc.olabel = 0;
|
||||
arc.weight = Weight::One();
|
||||
arc.nextstate = nextstate;
|
||||
fst->AddArc(curstate, arc);
|
||||
}
|
||||
curstate = nextstate;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
template<class Arc, class I>
|
||||
void CreateFactorFst(const std::vector<std::vector<I> > &sequences,
|
||||
MutableFst<Arc> *fst) {
|
||||
KALDI_ASSERT_IS_INTEGER_TYPE(I);
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
assert(fst != NULL);
|
||||
fst->DeleteStates();
|
||||
StateId loopstate = fst->AddState();
|
||||
assert(loopstate == 0);
|
||||
fst->SetStart(0);
|
||||
fst->SetFinal(0, Weight::One());
|
||||
if (sequences.size() != 0) assert(sequences[0].size() == 0); // can't replace epsilon...
|
||||
|
||||
for (Label olabel = 1; olabel < static_cast<Label>(sequences.size()); olabel++) {
|
||||
size_t len = sequences[olabel].size();
|
||||
if (len == 0) {
|
||||
Arc arc(0, olabel, Weight::One(), loopstate);
|
||||
fst->AddArc(loopstate, arc);
|
||||
} else {
|
||||
StateId curstate = loopstate;
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
StateId nextstate = (i == len-1 ? loopstate : fst->AddState());
|
||||
Arc arc(sequences[olabel][i], (i == 0 ? olabel : 0), Weight::One(), nextstate);
|
||||
fst->AddArc(curstate, arc);
|
||||
curstate = nextstate;
|
||||
}
|
||||
}
|
||||
}
|
||||
fst->SetProperties(kOLabelSorted, kOLabelSorted);
|
||||
}
|
||||
|
||||
|
||||
template<class Arc, class I>
|
||||
void CreateMapFst(const std::vector<I> &symbol_map,
|
||||
MutableFst<Arc> *fst) {
|
||||
KALDI_ASSERT_IS_INTEGER_TYPE(I);
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
assert(fst != NULL);
|
||||
fst->DeleteStates();
|
||||
StateId loopstate = fst->AddState();
|
||||
assert(loopstate == 0);
|
||||
fst->SetStart(0);
|
||||
fst->SetFinal(0, Weight::One());
|
||||
assert(symbol_map.empty() || symbol_map[0] == 0); // FST cannot map epsilon to something else.
|
||||
for (Label olabel = 1; olabel < static_cast<Label>(symbol_map.size()); olabel++) {
|
||||
Arc arc(symbol_map[olabel], olabel, Weight::One(), loopstate);
|
||||
fst->AddArc(loopstate, arc);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
} // end namespace fst.
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,191 @@
|
||||
// fstext/factor-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#include "fstext/factor.h"
|
||||
#include "fstext/fstext-utils.h"
|
||||
#include "fstext/fst-test-utils.h"
|
||||
#include "base/kaldi-math.h"
|
||||
|
||||
|
||||
namespace fst
|
||||
{
|
||||
using std::vector;
|
||||
|
||||
// Don't instantiate with log semiring, as RandEquivalent may fail.
|
||||
template<class Arc> static void TestFactor() {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> fst;
|
||||
int n_syms = 2 + kaldi::Rand() % 5, n_arcs = 5 + kaldi::Rand() % 30, n_final = 1 + kaldi::Rand()%10;
|
||||
|
||||
SymbolTable symtab("my-symbol-table"), *sptr = &symtab;
|
||||
|
||||
vector<Label> all_syms; // including epsilon.
|
||||
// Put symbols in the symbol table from 1..n_syms-1.
|
||||
for (size_t i = 0;i < (size_t)n_syms;i++) {
|
||||
std::stringstream ss;
|
||||
if (i == 0) ss << "<eps>";
|
||||
else ss<<i;
|
||||
Label cur_lab = sptr->AddSymbol(ss.str());
|
||||
assert(cur_lab == (Label)i);
|
||||
all_syms.push_back(cur_lab);
|
||||
}
|
||||
assert(all_syms[0] == 0);
|
||||
|
||||
fst.AddState();
|
||||
int cur_num_states = 1;
|
||||
for (int i = 0; i < n_arcs; i++) {
|
||||
StateId src_state = kaldi::Rand() % cur_num_states;
|
||||
StateId dst_state;
|
||||
if (kaldi::RandUniform() < 0.1) dst_state = kaldi::Rand() % cur_num_states;
|
||||
else {
|
||||
dst_state = cur_num_states++; fst.AddState();
|
||||
}
|
||||
Arc arc;
|
||||
if (kaldi::RandUniform() < 0.5) arc.ilabel = all_syms[kaldi::Rand()%all_syms.size()];
|
||||
else arc.ilabel = 0;
|
||||
if (kaldi::RandUniform() < 0.5) arc.olabel = all_syms[kaldi::Rand()%all_syms.size()];
|
||||
else arc.olabel = 0;
|
||||
arc.weight = (Weight) (0 + 0.1*(kaldi::Rand() % 5));
|
||||
arc.nextstate = dst_state;
|
||||
fst.AddArc(src_state, arc);
|
||||
}
|
||||
for (int i = 0; i < n_final; i++) {
|
||||
fst.SetFinal(kaldi::Rand() % cur_num_states, (Weight) (0 + 0.1*(kaldi::Rand() % 5)));
|
||||
}
|
||||
|
||||
if (kaldi::RandUniform() < 0.8) fst.SetStart(0); // usually leads to nicer examples.
|
||||
else fst.SetStart(kaldi::Rand() % cur_num_states);
|
||||
|
||||
std::cout <<" printing before trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
// Trim resulting FST.
|
||||
Connect(&fst);
|
||||
|
||||
std::cout <<" printing after trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
if (fst.Start() == kNoStateId) return; // "Connect" made it empty.
|
||||
|
||||
VectorFst<Arc> fst_pushed;
|
||||
Push<Arc, REWEIGHT_TO_INITIAL>(fst, &fst_pushed, kPushLabels);
|
||||
|
||||
VectorFst<Arc> fst_factored;
|
||||
vector<vector<typename Arc::Label> > symbols;
|
||||
|
||||
Factor(fst, &fst_factored, &symbols);
|
||||
|
||||
// Check no epsilons in "symbols".
|
||||
for (size_t i = 0; i < symbols.size(); i++)
|
||||
assert(symbols[i].size() == 0 || *(std::min(symbols[i].begin(), symbols[i].end())) > 0);
|
||||
|
||||
VectorFst<Arc> fst_factored_pushed;
|
||||
vector<vector<typename Arc::Label> > symbols_pushed;
|
||||
Factor(fst_pushed, &fst_factored_pushed, &symbols_pushed);
|
||||
|
||||
std::cout << "Unfactored has "<<fst.NumStates()<<" states, factored has "<<fst_factored.NumStates()<<", and pushed+factored has "<<fst_factored_pushed.NumStates()<<'\n';
|
||||
|
||||
assert(fst_factored.NumStates() <= fst.NumStates());
|
||||
// assert(fst_factored_pushed.NumStates() <= fst_factored.NumStates()); // pushing should only help. [ no, it doesn't]
|
||||
assert(fst_factored_pushed.NumStates() <= fst_pushed.NumStates());
|
||||
|
||||
VectorFst<Arc> fst_factored_copy(fst_factored);
|
||||
|
||||
VectorFst<Arc> fst_factored_unfactored(fst_factored);
|
||||
ExpandInputSequences(symbols, &fst_factored_unfactored);
|
||||
|
||||
VectorFst<Arc> factor_fst;
|
||||
CreateFactorFst(symbols, &factor_fst);
|
||||
VectorFst<Arc> fst_factored_unfactored2;
|
||||
Compose(factor_fst, fst_factored, &fst_factored_unfactored2);
|
||||
|
||||
ExpandInputSequences(symbols_pushed, &fst_factored_pushed);
|
||||
|
||||
assert(RandEquivalent(fst, fst_factored_unfactored, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
assert(RandEquivalent(fst, fst_factored_unfactored2, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
assert(RandEquivalent(fst, fst_factored_pushed, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
{ // Have tested for equivalence; now do another test: that FactorFst actually finds all
|
||||
// the factors. Do this by inserting factors using ExpandInputSequences and making sure it gets
|
||||
// rid of them all.
|
||||
Label max_label = *(std::max_element(all_syms.begin(), all_syms.end()));
|
||||
vector<vector<Label> > new_labels(max_label+1);
|
||||
for (Label l = 1; l < static_cast<Label>(new_labels.size()); l++) {
|
||||
int n = kaldi::Rand() % 5;
|
||||
for (int i = 0; i < n; i++) new_labels[l].push_back(kaldi::Rand() % 100);
|
||||
}
|
||||
VectorFst<Arc> fst_expanded(fst);
|
||||
ExpandInputSequences(new_labels, &fst_expanded);
|
||||
|
||||
vector<vector<Label> > factors;
|
||||
VectorFst<Arc> fst_reduced;
|
||||
Factor(fst_expanded, &fst_reduced, &factors);
|
||||
assert(fst_reduced.NumStates() <= fst.NumStates()); // Checking that it found all the factors.
|
||||
}
|
||||
|
||||
{ // This block test MapInputSymbols [but relies on the correctness of Factor
|
||||
// and ExpandInputSequences to do so].
|
||||
|
||||
std::map<Label, Label> symbols_reverse_map; // from new->old.
|
||||
symbols_reverse_map[0] = 0; // map eps to eps.
|
||||
for (Label i = 1; i < static_cast<Label>(symbols.size()); i++) {
|
||||
Label new_i;
|
||||
do {
|
||||
new_i = kaldi::Rand() % (symbols.size() + 20);
|
||||
} while (symbols_reverse_map.count(new_i) == 1);
|
||||
symbols_reverse_map[new_i] = i;
|
||||
}
|
||||
vector<vector<Label> > symbols_new;
|
||||
vector<Label> symbol_map(symbols.size()); // from old->new.
|
||||
typename std::map<Label, Label>::iterator iter = symbols_reverse_map.begin();
|
||||
for (; iter != symbols_reverse_map.end(); iter++) {
|
||||
Label new_label = iter->first, old_label = iter->second;
|
||||
if (new_label >= static_cast<Label>(symbols_new.size())) symbols_new.resize(new_label+1);
|
||||
symbols_new[new_label] = symbols[old_label];
|
||||
symbol_map[old_label] = new_label;
|
||||
}
|
||||
MapInputSymbols(symbol_map, &fst_factored_copy);
|
||||
ExpandInputSequences(symbols_new, &fst_factored_copy);
|
||||
assert(RandEquivalent(fst, fst_factored_copy,
|
||||
5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/,
|
||||
100/*path length-- max?*/));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
} // namespace fst
|
||||
|
||||
int main() {
|
||||
using namespace fst;
|
||||
for (int i = 0;i < 25;i++) {
|
||||
TestFactor<fst::StdArc>();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
// fstext/factor.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_FACTOR_H_
|
||||
#define KALDI_FSTEXT_FACTOR_H_
|
||||
|
||||
/*
|
||||
This header declares the Factor function, which takes an FST and
|
||||
compresses it by detecting linear chains of states, and creating
|
||||
special input symbols that represent these chains. It outputs enough
|
||||
information to be able to reconstruct the original sequences [i.e.
|
||||
the mapping between the new symbols, and sequences of the original
|
||||
symbols]. It ensures that the original symbols all have the same
|
||||
number as a corresponding "new" symbol representing a sequence of length
|
||||
one; this enables certain optimizations later on.
|
||||
*/
|
||||
|
||||
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
#include "util/const-integer-set.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
/**
|
||||
Factor identifies linear chains of states with an olabel (if any)
|
||||
only on the first arc of the chain, and possibly a sequence of
|
||||
ilabels; it outputs an FST with different symbols on the input
|
||||
that represent sequences of the original input symbols; it outputs
|
||||
the mapping from the new symbol to sequences of original symbols,
|
||||
as "symbols" [zero is reserved for epsilon].
|
||||
|
||||
As a side effect it also sorts the FST in depth-first order. Factor will
|
||||
usually do the best job when the olabels have been pushed to the left,
|
||||
i.e. if you make a call like
|
||||
|
||||
Push<Arc, REWEIGHT_TO_INITIAL>(fsta, &fstb, kPushLabels);
|
||||
|
||||
This is because it only creates a chain with olabels on the first arc of the
|
||||
chain (or a chain with no olabels). [it's possible to construct cases where
|
||||
pushing makes things worse, though]. After Factor, the composition of *ofst
|
||||
with the result of calling CreateFactorFst(*symbols) should be equivalent to
|
||||
fst. Alternatively, calling ExpandInputSequences with ofst and *symbols
|
||||
would produce something equivalent to fst.
|
||||
*/
|
||||
|
||||
template<class Arc, class I>
|
||||
void Factor(const Fst<Arc> &fst, MutableFst<Arc> *ofst,
|
||||
std::vector<std::vector<I> > *symbols);
|
||||
|
||||
|
||||
/// This is a more conventional interface of Factor that outputs
|
||||
/// the result as two FSTs.
|
||||
template<class Arc>
|
||||
void Factor(const Fst<Arc> &fst, MutableFst<Arc> *ofst1,
|
||||
MutableFst<Arc> *ofst2);
|
||||
|
||||
|
||||
|
||||
/// ExpandInputSequences expands out the input symbols into sequences of input
|
||||
/// symbols. It creates linear chains of states for each arc that had >1
|
||||
/// augmented symbol on it. It also sets the input symbol table to NULL, since
|
||||
/// in case you did have a symbol table there it would no longer be valid. It
|
||||
/// leaves any weight and output symbols on the first arc of the chain.
|
||||
template<class Arc, class I>
|
||||
void ExpandInputSequences(const std::vector<std::vector<I> > &sequences,
|
||||
MutableFst<Arc> *fst);
|
||||
|
||||
|
||||
/// The function CreateFactorFst will create an FST that expands out the
|
||||
/// "factors" that are the indices of the "sequences" array, into linear sequences
|
||||
/// of symbols. There is a single start and end state (state 0), and for each
|
||||
/// nonzero index i into the array "sequences", there is an arc from state 0 that
|
||||
/// has output-label i, and enters a chain of states with output epsilons and input
|
||||
/// labels corresponding to the remaining elements of the sequences, terminating
|
||||
/// again in state 0. This FST is output-deterministic and sorted on olabel.
|
||||
/// Composing an FST on the left with the output of this function, should be the
|
||||
/// same as calling "ExpandInputSequences". Use TableCompose (see table-matcher.h)
|
||||
/// for efficiency.
|
||||
template<class Arc, class I>
|
||||
void CreateFactorFst(const std::vector<std::vector<I> > &sequences,
|
||||
MutableFst<Arc> *fst);
|
||||
|
||||
|
||||
/// CreateMapFst will create an FST representing this symbol_map. The
|
||||
/// FST has a single loop state with single-arc loops with
|
||||
/// isymbol = symbol_map[i], osymbol = i. The resulting FST applies this
|
||||
/// map to the input symbols of something we compose with it on the right.
|
||||
/// Must have symbol_map[0] == 0.
|
||||
template<class Arc, class I>
|
||||
void CreateMapFst(const std::vector<I> &symbol_map,
|
||||
MutableFst<Arc> *fst);
|
||||
|
||||
|
||||
enum StatePropertiesEnum
|
||||
{ kStateFinal = 0x1,
|
||||
kStateInitial = 0x2,
|
||||
kStateArcsIn = 0x4,
|
||||
kStateMultipleArcsIn = 0x8,
|
||||
kStateArcsOut = 0x10,
|
||||
kStateMultipleArcsOut = 0x20,
|
||||
kStateOlabelsOut = 0x40,
|
||||
kStateIlabelsOut = 0x80 };
|
||||
|
||||
typedef unsigned char StatePropertiesType;
|
||||
|
||||
/**
|
||||
This function works out various properties of the states in the
|
||||
FST, using the bit properties defined in StatePropertiesEnum. */
|
||||
template<class Arc>
|
||||
void GetStateProperties(const Fst<Arc> &fst,
|
||||
typename Arc::StateId max_state,
|
||||
std::vector<StatePropertiesType> *props);
|
||||
|
||||
|
||||
|
||||
template<class Arc>
|
||||
class DfsOrderVisitor {
|
||||
// visitor class that gives the user the dfs order,
|
||||
// c.f. dfs-visit.h. Used in factor-fst-impl.h
|
||||
typedef typename Arc::StateId StateId;
|
||||
public:
|
||||
DfsOrderVisitor(std::vector<StateId> *order): order_(order) { order->clear(); }
|
||||
void InitVisit(const Fst<Arc> &fst) {}
|
||||
bool InitState(StateId s, StateId) { order_->push_back(s); return true; }
|
||||
bool TreeArc(StateId, const Arc&) { return true; }
|
||||
bool BackArc(StateId, const Arc&) { return true; }
|
||||
bool ForwardOrCrossArc(StateId, const Arc&) { return true; }
|
||||
void FinishState(StateId, StateId, const Arc *) { }
|
||||
void FinishVisit() { }
|
||||
private:
|
||||
std::vector<StateId> *order_;
|
||||
};
|
||||
|
||||
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#include "factor-inl.h"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,33 @@
|
||||
// fstext/fst-test-utils.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_FST_TEST_UTILS_H_
|
||||
#define KALDI_FSTEXT_FST_TEST_UTILS_H_
|
||||
|
||||
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
// Just some #includes.
|
||||
#include "fst/script/print-impl.h"
|
||||
#include "fstext/rand-fst.h"
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
// fstext/fstext-lib.h
|
||||
|
||||
// Copyright 2009-2012 Microsoft Corporation Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_FSTEXT_LIB_H_
|
||||
#define KALDI_FSTEXT_FSTEXT_LIB_H_
|
||||
#include "fst/fstlib.h"
|
||||
#include "fstext/context-fst.h"
|
||||
#include "fstext/determinize-star.h"
|
||||
#include "fstext/factor.h"
|
||||
#include "fstext/fst-test-utils.h"
|
||||
#include "fstext/fstext-utils.h"
|
||||
#include "fstext/pre-determinize.h"
|
||||
#include "fstext/table-matcher.h"
|
||||
#include "fstext/trivial-factor-weight.h"
|
||||
#include "fstext/lattice-weight.h"
|
||||
#include "fstext/lattice-utils.h"
|
||||
#include "fstext/determinize-lattice.h"
|
||||
#include "fstext/deterministic-fst.h"
|
||||
#include "fstext/kaldi-fst-io.h"
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,434 @@
|
||||
// fstext/fstext-utils-test.cc
|
||||
|
||||
// Copyright 2009-2012 Microsoft Corporation Daniel Povey
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "base/kaldi-common.h" // for exceptions
|
||||
#include "fstext/fstext-utils.h"
|
||||
#include "fstext/fst-test-utils.h"
|
||||
#include "util/stl-utils.h"
|
||||
#include "base/kaldi-math.h"
|
||||
|
||||
namespace fst
|
||||
{
|
||||
using std::vector;
|
||||
using std::cout;
|
||||
|
||||
template<class Arc, class I>
|
||||
void TestMakeLinearAcceptor() {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
int len = kaldi::Rand() % 10;
|
||||
vector<I> vec;
|
||||
vector<I> vec_nozeros;
|
||||
for (int i = 0; i < len; i++) {
|
||||
int j = kaldi::Rand() % len;
|
||||
vec.push_back(j);
|
||||
if (j != 0) vec_nozeros.push_back(j);
|
||||
}
|
||||
|
||||
|
||||
VectorFst<Arc> vfst;
|
||||
MakeLinearAcceptor(vec, &vfst);
|
||||
vector<I> vec2;
|
||||
vector<I> vec3;
|
||||
Weight w;
|
||||
GetLinearSymbolSequence(vfst, &vec2, &vec3, &w);
|
||||
assert(w == Weight::One());
|
||||
assert(vec_nozeros == vec2);
|
||||
assert(vec_nozeros == vec3);
|
||||
|
||||
if (vec2.size() != 0 || vec3.size() != 0) { // This test might not work
|
||||
// for empty sequences...
|
||||
{
|
||||
vector<VectorFst<Arc> > fstvec;
|
||||
NbestAsFsts(vfst, 1, &fstvec);
|
||||
KALDI_ASSERT(fstvec.size() == 1);
|
||||
assert(RandEquivalent(vfst, fstvec[0], 2/*paths*/, 0.01/*delta*/,
|
||||
kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
}
|
||||
}
|
||||
bool include_eps = (kaldi::Rand() % 2 == 0);
|
||||
if (!include_eps) vec = vec_nozeros;
|
||||
kaldi::SortAndUniq(&vec);
|
||||
|
||||
vector<I> vec4;
|
||||
GetInputSymbols(vfst, include_eps, &vec4);
|
||||
assert(vec4 == vec);
|
||||
vector<I> vec5;
|
||||
GetInputSymbols(vfst, include_eps, &vec5);
|
||||
}
|
||||
|
||||
|
||||
template<class Arc> void TestDeterminizeStarInLog() {
|
||||
VectorFst<Arc> *fst = RandFst<Arc>();
|
||||
VectorFst<Arc> fst_copy(fst);
|
||||
typename Arc::Label next_sym = 1 + HighestNumberedInputSymbol(*fst);
|
||||
vector<typename Arc::Label> syms;
|
||||
PreDeterminize(fst, NULL, "#", next_sym, &syms);
|
||||
|
||||
|
||||
}
|
||||
|
||||
// Don't instantiate with log semiring, as RandEquivalent may fail.
|
||||
template<class Arc> void TestSafeDeterminizeWrapper() { // also tests SafeDeterminizeMinimizeWrapper().
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> *fst = new VectorFst<Arc>();
|
||||
int n_syms = 2 + kaldi::Rand() % 5, n_states = 3 + kaldi::Rand() % 10, n_arcs = 5 + kaldi::Rand() % 30, n_final = 1 + kaldi::Rand()%3; // Up to 2 unique symbols.
|
||||
cout << "Testing pre-determinize with "<<n_syms<<" symbols, "<<n_states<<" states and "<<n_arcs<<" arcs and "<<n_final<<" final states.\n";
|
||||
SymbolTable *sptr = new SymbolTable("my-symbol-table");
|
||||
sptr->AddSymbol("<eps>");
|
||||
delete sptr;
|
||||
sptr = new SymbolTable("my-symbol-table");
|
||||
|
||||
vector<Label> all_syms; // including epsilon.
|
||||
// Put symbols in the symbol table from 1..n_syms-1.
|
||||
for (size_t i = 0;i < (size_t)n_syms;i++) {
|
||||
std::stringstream ss;
|
||||
if (i == 0) ss << "<eps>";
|
||||
else ss<<i;
|
||||
Label cur_lab = sptr->AddSymbol(ss.str());
|
||||
assert(cur_lab == (Label)i);
|
||||
all_syms.push_back(cur_lab);
|
||||
}
|
||||
assert(all_syms[0] == 0);
|
||||
|
||||
// Create states.
|
||||
vector<StateId> all_states;
|
||||
for (size_t i = 0;i < (size_t)n_states;i++) {
|
||||
StateId this_state = fst->AddState();
|
||||
if (i == 0) fst->SetStart(i);
|
||||
all_states.push_back(this_state);
|
||||
}
|
||||
// Set final states.
|
||||
for (size_t j = 0;j < (size_t)n_final;j++) {
|
||||
StateId id = all_states[kaldi::Rand() % n_states];
|
||||
Weight weight = (Weight)(0.33*(kaldi::Rand() % 5) );
|
||||
printf("calling SetFinal with %d and %f\n", id, weight.Value());
|
||||
fst->SetFinal(id, weight);
|
||||
}
|
||||
// Create arcs.
|
||||
for (size_t i = 0;i < (size_t)n_arcs;i++) {
|
||||
Arc a;
|
||||
a.nextstate = all_states[kaldi::Rand() % n_states];
|
||||
a.ilabel = all_syms[kaldi::Rand() % n_syms];
|
||||
a.olabel = all_syms[kaldi::Rand() % n_syms]; // same input+output vocab.
|
||||
a.weight = (Weight) (0.33*(kaldi::Rand() % 2));
|
||||
StateId start_state = all_states[kaldi::Rand() % n_states];
|
||||
fst->AddArc(start_state, a);
|
||||
}
|
||||
|
||||
std::cout <<" printing before trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
// Trim resulting FST.
|
||||
Connect(fst);
|
||||
|
||||
std::cout <<" printing after trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
VectorFst<Arc> *fst_copy_orig = new VectorFst<Arc>(*fst);
|
||||
|
||||
VectorFst<Arc> *fst_det = new VectorFst<Arc>;
|
||||
|
||||
vector<Label> extra_syms;
|
||||
if (fst->Start() != kNoStateId) { // "Connect" did not make it empty....
|
||||
if (kaldi::Rand() % 2 == 0)
|
||||
SafeDeterminizeWrapper(fst_copy_orig, fst_det);
|
||||
else {
|
||||
if (kaldi::Rand() % 2 == 0)
|
||||
SafeDeterminizeMinimizeWrapper(fst_copy_orig, fst_det);
|
||||
else
|
||||
SafeDeterminizeMinimizeWrapperInLog(fst_copy_orig, fst_det);
|
||||
}
|
||||
|
||||
// no because does shortest-dist on weights even if not pushing on them.
|
||||
// PushInLog<REWEIGHT_TO_INITIAL>(fst_det, kPushLabels); // will always succeed.
|
||||
KALDI_LOG << "Num states [orig]: " << fst->NumStates() << "[det]" << fst_det->NumStates();
|
||||
assert(RandEquivalent(*fst, *fst_det, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
}
|
||||
delete fst;
|
||||
delete fst_copy_orig;
|
||||
delete fst_det;
|
||||
delete sptr;
|
||||
}
|
||||
|
||||
|
||||
// Don't instantiate with log semiring, as RandEquivalent may fail.
|
||||
void TestPushInLog() { // also tests SafeDeterminizeMinimizeWrapper().
|
||||
typedef StdArc Arc;
|
||||
typedef Arc::Label Label;
|
||||
typedef Arc::StateId StateId;
|
||||
typedef Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> *fst = RandFst<Arc>();
|
||||
VectorFst<Arc> fst2(*fst);
|
||||
PushInLog<REWEIGHT_TO_INITIAL>(&fst2, kPushLabels|kPushWeights, 0.01); // speed it up using large delta.
|
||||
assert(RandEquivalent(*fst, fst2, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
delete fst;
|
||||
}
|
||||
|
||||
|
||||
|
||||
template<class Arc> void TestAcceptorMinimize() {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> *fst = RandFst<Arc>();
|
||||
|
||||
Project(fst, PROJECT_INPUT);
|
||||
RemoveWeights(fst);
|
||||
|
||||
VectorFst<Arc> fst2(*fst);
|
||||
internal::AcceptorMinimize(&fst2);
|
||||
|
||||
assert(RandEquivalent(*fst, fst2, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
delete fst;
|
||||
}
|
||||
|
||||
|
||||
template<class Arc> void TestMakeSymbolsSame() {
|
||||
|
||||
VectorFst<Arc> *fst = RandFst<Arc>();
|
||||
bool foll = (kaldi::Rand() % 2 == 0);
|
||||
bool is_symbol = (kaldi::Rand() % 2 == 0);
|
||||
|
||||
|
||||
VectorFst<Arc> fst2(*fst);
|
||||
|
||||
if (foll) {
|
||||
MakeFollowingInputSymbolsSame(is_symbol, &fst2);
|
||||
assert(FollowingInputSymbolsAreSame(is_symbol, fst2));
|
||||
} else {
|
||||
MakePrecedingInputSymbolsSame(is_symbol, &fst2);
|
||||
assert(PrecedingInputSymbolsAreSame(is_symbol, fst2));
|
||||
}
|
||||
|
||||
|
||||
assert(RandEquivalent(*fst, fst2, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
delete fst;
|
||||
}
|
||||
|
||||
|
||||
template<class Arc>
|
||||
struct TestFunctor {
|
||||
typedef int32 Result;
|
||||
typedef typename Arc::Label Arg;
|
||||
Result operator () (Arg a) const {
|
||||
if (a == kNoLabel) return -1;
|
||||
else if (a == 0) return 0;
|
||||
else {
|
||||
return 1 + ((a-1) % 10);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template<class Arc> void TestMakeSymbolsSameClass() {
|
||||
|
||||
VectorFst<Arc> *fst = RandFst<Arc>();
|
||||
bool foll = (kaldi::Rand() % 2 == 0);
|
||||
bool is_symbol = (kaldi::Rand() % 2 == 0);
|
||||
|
||||
|
||||
VectorFst<Arc> fst2(*fst);
|
||||
|
||||
TestFunctor<Arc> f;
|
||||
if (foll) {
|
||||
MakeFollowingInputSymbolsSameClass(is_symbol, &fst2, f);
|
||||
assert(FollowingInputSymbolsAreSameClass(is_symbol, fst2, f));
|
||||
} else {
|
||||
MakePrecedingInputSymbolsSameClass(is_symbol, &fst2, f);
|
||||
assert(PrecedingInputSymbolsAreSameClass(is_symbol, fst2, f));
|
||||
}
|
||||
|
||||
assert(RandEquivalent(*fst, fst2, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
delete fst;
|
||||
}
|
||||
|
||||
|
||||
// MakeLoopFstCompare is as MakeLoopFst but implmented differently [ less efficiently
|
||||
// but more clearly], so we can check for equivalence.
|
||||
template<class Arc>
|
||||
VectorFst<Arc>* MakeLoopFstCompare(const vector<const ExpandedFst<Arc> *> &fsts) {
|
||||
VectorFst<Arc> *ans = new VectorFst<Arc>;
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
for (Label i = 0; i < fsts.size(); i++) {
|
||||
if (fsts[i] != NULL) {
|
||||
VectorFst<Arc> i_fst; // accepts symbol i on output.
|
||||
i_fst.AddState(); i_fst.AddState();
|
||||
i_fst.SetStart(0); i_fst.SetFinal(1, Weight::One());
|
||||
i_fst.AddArc(0, Arc(0, i, Weight::One(), 1));
|
||||
VectorFst<Arc> other_fst(*(fsts[i])); // copy it.
|
||||
ClearSymbols(false, true, &other_fst); // Clear output symbols so symbols
|
||||
// are on input side.
|
||||
Concat(&i_fst, other_fst); // now i_fst is "i_fst [concat] other_fst".
|
||||
Union(ans, i_fst);
|
||||
}
|
||||
}
|
||||
Closure(ans, CLOSURE_STAR);
|
||||
return ans;
|
||||
}
|
||||
|
||||
|
||||
template<class Arc> void TestMakeLoopFst() {
|
||||
|
||||
int num_fsts = kaldi::Rand() % 10;
|
||||
vector<const ExpandedFst<Arc>* > fsts(num_fsts, (const ExpandedFst<Arc>*)NULL);
|
||||
for (int i = 0; i < num_fsts; i++) {
|
||||
if (kaldi::Rand() % 2 == 0) { // put an fst there.
|
||||
VectorFst<Arc> *fst = RandFst<Arc>();
|
||||
Project(fst, PROJECT_INPUT); // make input & output labels the same.
|
||||
fsts[i] = fst;
|
||||
} else { // this is to test that it works with the caching.
|
||||
fsts[i] = fsts[i/2];
|
||||
}
|
||||
}
|
||||
|
||||
VectorFst<Arc> *fst1 = MakeLoopFst(fsts),
|
||||
*fst2 = MakeLoopFstCompare(fsts);
|
||||
|
||||
assert(fst1->Properties(kOLabelSorted, kOLabelSorted) != 0);
|
||||
|
||||
assert(RandEquivalent(*fst1, *fst2, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
delete fst1;
|
||||
delete fst2;
|
||||
std::sort(fsts.begin(), fsts.end());
|
||||
fsts.erase(std::unique(fsts.begin(), fsts.end()), fsts.end());
|
||||
for (int i = 0; i < (int)fsts.size(); i++)
|
||||
delete fsts[i];
|
||||
}
|
||||
|
||||
|
||||
|
||||
template<class Arc>
|
||||
void TestEqualAlign() {
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
RandFstOptions opts;
|
||||
opts.allow_empty = false;
|
||||
VectorFst<Arc> *fst = RandFst<Arc>();
|
||||
int length = 10 + kaldi::Rand() % 20;
|
||||
|
||||
VectorFst<Arc> fst_path;
|
||||
if (EqualAlign(*fst, length, kaldi::Rand(), &fst_path)) {
|
||||
std::cout << "EqualAlign succeeded\n";
|
||||
vector<int32> isymbol_seq, osymbol_seq;
|
||||
typename Arc::Weight weight;
|
||||
GetLinearSymbolSequence(fst_path, &isymbol_seq, &osymbol_seq, &weight);
|
||||
assert(isymbol_seq.size() == length);
|
||||
Invert(&fst_path);
|
||||
VectorFst<Arc> fst_composed;
|
||||
Compose(fst_path, *fst, &fst_composed);
|
||||
assert(fst_composed.Start() != kNoStateId); // make sure nonempty.
|
||||
} else {
|
||||
std::cout << "EqualAlign did not generate alignment\n";
|
||||
}
|
||||
delete fst;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template<class Arc> void Print(const Fst<Arc> &fst, std::string message) {
|
||||
std::cout << message << "\n";
|
||||
FstPrinter<Arc> fstprinter(fst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
template<class Arc>
|
||||
void TestRemoveUselessArcs() {
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
RandFstOptions opts;
|
||||
opts.allow_empty = false;
|
||||
VectorFst<Arc> *fst = RandFst<Arc>();
|
||||
// Print(*fst, "[testremoveuselessarcs]:fst:");
|
||||
UniformArcSelector<Arc> selector;
|
||||
RandGenOptions<UniformArcSelector<Arc> > randgen_opts(selector);
|
||||
VectorFst<Arc> fst_path;
|
||||
RandGen(*fst, &fst_path, randgen_opts);
|
||||
Project(&fst_path, PROJECT_INPUT);
|
||||
// Print(fst_path, "[testremoveuselessarcs]:fstpath:");
|
||||
|
||||
VectorFst<Arc> fst_nouseless(*fst);
|
||||
RemoveUselessArcs(&fst_nouseless);
|
||||
// Print(fst_nouseless, "[testremoveuselessarcs]:fst_nouseless:");
|
||||
|
||||
VectorFst<Arc> orig_composed,
|
||||
nouseless_composed;
|
||||
Compose(fst_path, *fst, &orig_composed);
|
||||
Compose(fst_path, fst_nouseless, &nouseless_composed);
|
||||
|
||||
// Print(orig_composed, "[testremoveuselessarcs]:orig_composed");
|
||||
// Print(nouseless_composed, "[testremoveuselessarcs]:nouseless_composed");
|
||||
|
||||
VectorFst<Arc> orig_bestpath,
|
||||
nouseless_bestpath;
|
||||
ShortestPath(orig_composed, &orig_bestpath);
|
||||
ShortestPath(nouseless_composed, &nouseless_bestpath);
|
||||
// Print(orig_bestpath, "[testremoveuselessarcs]:orig_bestpath");
|
||||
// Print(nouseless_bestpath, "[testremoveuselessarcs]:nouseless_bestpath");
|
||||
|
||||
typename Arc::Weight worig, wnouseless;
|
||||
GetLinearSymbolSequence<Arc, int>(orig_bestpath, NULL, NULL, &worig);
|
||||
GetLinearSymbolSequence<Arc, int>(nouseless_bestpath, NULL, NULL, &wnouseless);
|
||||
assert(ApproxEqual(worig, wnouseless, kDelta));
|
||||
|
||||
// assert(RandEquivalent(orig_bestpath, nouseless_bestpath, 5/*paths*/, 0.01/*delta*/, Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
delete fst;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
|
||||
int main() {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
fst::TestMakeLinearAcceptor<fst::StdArc, int>(); // this also tests GetLinearSymbolSequence, GetInputSymbols and GetOutputSymbols.
|
||||
fst::TestMakeLinearAcceptor<fst::StdArc, int32>();
|
||||
fst::TestMakeLinearAcceptor<fst::StdArc, uint32>();
|
||||
fst::TestSafeDeterminizeWrapper<fst::StdArc>();
|
||||
fst::TestAcceptorMinimize<fst::StdArc>();
|
||||
fst::TestMakeSymbolsSame<fst::StdArc>();
|
||||
fst::TestMakeSymbolsSame<fst::LogArc>();
|
||||
fst::TestMakeSymbolsSameClass<fst::StdArc>();
|
||||
fst::TestMakeSymbolsSameClass<fst::LogArc>();
|
||||
fst::TestMakeLoopFst<fst::StdArc>();
|
||||
fst::TestMakeLoopFst<fst::LogArc>();
|
||||
fst::TestEqualAlign<fst::StdArc>();
|
||||
fst::TestEqualAlign<fst::LogArc>();
|
||||
fst::TestRemoveUselessArcs<fst::StdArc>();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,427 @@
|
||||
// fstext/fstext-utils.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2012-2013 Johns Hopkins University (Author: Daniel Povey)
|
||||
// 2013 Guoguo Chen
|
||||
// 2014 Telepoint Global Hosting Service, LLC. (Author: David Snyder)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_FSTEXT_UTILS_H_
|
||||
#define KALDI_FSTEXT_FSTEXT_UTILS_H_
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
#include "fstext/determinize-star.h"
|
||||
#include "fstext/remove-eps-local.h"
|
||||
#include "base/kaldi-common.h" // for error reporting macros.
|
||||
#include "util/text-utils.h" // for SplitStringToVector
|
||||
#include "fst/script/print-impl.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
/// Returns the highest numbered output symbol id of the FST (or zero
|
||||
/// for an empty FST.
|
||||
template<class Arc>
|
||||
typename Arc::Label HighestNumberedOutputSymbol(const Fst<Arc> &fst);
|
||||
|
||||
/// Returns the highest numbered input symbol id of the FST (or zero
|
||||
/// for an empty FST.
|
||||
template<class Arc>
|
||||
typename Arc::Label HighestNumberedInputSymbol(const Fst<Arc> &fst);
|
||||
|
||||
/// Returns the total number of arcs in an FST.
|
||||
template<class Arc>
|
||||
typename Arc::StateId NumArcs(const ExpandedFst<Arc> &fst);
|
||||
|
||||
/// GetInputSymbols gets the list of symbols on the input of fst
|
||||
/// (including epsilon, if include_eps == true), as a sorted, unique
|
||||
/// list.
|
||||
template<class Arc, class I>
|
||||
void GetInputSymbols(const Fst<Arc> &fst,
|
||||
bool include_eps,
|
||||
std::vector<I> *symbols);
|
||||
|
||||
/// GetOutputSymbols gets the list of symbols on the output of fst
|
||||
/// (including epsilon, if include_eps == true)
|
||||
template<class Arc, class I>
|
||||
void GetOutputSymbols(const Fst<Arc> &fst,
|
||||
bool include_eps,
|
||||
std::vector<I> *symbols);
|
||||
|
||||
/// ClearSymbols sets all the symbols on the input and/or
|
||||
/// output side of the FST to zero, as specified.
|
||||
/// It does not alter the symbol tables.
|
||||
template<class Arc>
|
||||
void ClearSymbols(bool clear_input,
|
||||
bool clear_output,
|
||||
MutableFst<Arc> *fst);
|
||||
|
||||
template<class I>
|
||||
void GetSymbols(const SymbolTable &symtab,
|
||||
bool include_eps,
|
||||
std::vector<I> *syms_out);
|
||||
|
||||
|
||||
|
||||
inline
|
||||
void DeterminizeStarInLog(VectorFst<StdArc> *fst, float delta = kDelta, bool *debug_ptr = NULL,
|
||||
int max_states = -1);
|
||||
|
||||
|
||||
// e.g. of using this function: PushInLog<REWEIGHT_TO_INITIAL>(fst, kPushWeights|kPushLabels);
|
||||
|
||||
template<ReweightType rtype> // == REWEIGHT_TO_{INITIAL, FINAL}
|
||||
void PushInLog(VectorFst<StdArc> *fst, uint32 ptype, float delta = kDelta) {
|
||||
|
||||
// PushInLog pushes the FST
|
||||
// and returns a new pushed FST (labels and weights pushed to the left).
|
||||
VectorFst<LogArc> *fst_log = new VectorFst<LogArc>; // Want to determinize in log semiring.
|
||||
Cast(*fst, fst_log);
|
||||
VectorFst<StdArc> tmp;
|
||||
*fst = tmp; // free up memory.
|
||||
VectorFst<LogArc> *fst_pushed_log = new VectorFst<LogArc>;
|
||||
Push<LogArc, rtype>(*fst_log, fst_pushed_log, ptype, delta);
|
||||
Cast(*fst_pushed_log, fst);
|
||||
delete fst_log;
|
||||
delete fst_pushed_log;
|
||||
}
|
||||
|
||||
// Minimizes after encoding; applicable to all FSTs. It is like what you get
|
||||
// from the Minimize() function, except it will not push the weights, or the
|
||||
// symbols. This is better for our recipes, as we avoid ever pushing the
|
||||
// weights. However, it will only minimize optimally if your graphs are such
|
||||
// that the symbols are as far to the left as they can go, and the weights
|
||||
// in combinable paths are the same... hard to formalize this, but it's something
|
||||
// that is satisified by our normal FSTs.
|
||||
template<class Arc>
|
||||
void MinimizeEncoded(VectorFst<Arc> *fst, float delta = kDelta) {
|
||||
|
||||
Map(fst, QuantizeMapper<Arc>(delta));
|
||||
EncodeMapper<Arc> encoder(kEncodeLabels | kEncodeWeights, ENCODE);
|
||||
Encode(fst, &encoder);
|
||||
internal::AcceptorMinimize(fst);
|
||||
Decode(fst, encoder);
|
||||
}
|
||||
|
||||
|
||||
/// GetLinearSymbolSequence gets the symbol sequence from a linear FST.
|
||||
/// If the FST is not just a linear sequence, it returns false. If it is
|
||||
/// a linear sequence (including the empty FST), it returns true. In this
|
||||
/// case it outputs the symbol
|
||||
/// sequences as "isymbols_out" and "osymbols_out" (removing epsilons), and
|
||||
/// the total weight as "tot_weight". The total weight will be Weight::Zero()
|
||||
/// if the FST is empty. If any of the output pointers are NULL, it does not
|
||||
/// create that output.
|
||||
template<class Arc, class I>
|
||||
bool GetLinearSymbolSequence(const Fst<Arc> &fst,
|
||||
std::vector<I> *isymbols_out,
|
||||
std::vector<I> *osymbols_out,
|
||||
typename Arc::Weight *tot_weight_out);
|
||||
|
||||
|
||||
/// This function converts an FST with a special structure, which is
|
||||
/// output by the OpenFst functions ShortestPath and RandGen, and converts
|
||||
/// them into a std::vector of separate FSTs. This special structure is that
|
||||
/// the only state that has more than one (arcs-out or final-prob) is the
|
||||
/// start state. fsts_out is resized to the appropriate size.
|
||||
template<class Arc>
|
||||
void ConvertNbestToVector(const Fst<Arc> &fst,
|
||||
std::vector<VectorFst<Arc> > *fsts_out);
|
||||
|
||||
|
||||
/// Takes the n-shortest-paths (using ShortestPath), but outputs
|
||||
/// the result as a vector of up to n fsts. This function will
|
||||
/// size the "fsts_out" vector to however many paths it got
|
||||
/// (which will not exceed n). n must be >= 1.
|
||||
template<class Arc>
|
||||
void NbestAsFsts(const Fst<Arc> &fst,
|
||||
size_t n,
|
||||
std::vector<VectorFst<Arc> > *fsts_out);
|
||||
|
||||
|
||||
|
||||
|
||||
/// Creates unweighted linear acceptor from symbol sequence.
|
||||
template<class Arc, class I>
|
||||
void MakeLinearAcceptor(const std::vector<I> &labels, MutableFst<Arc> *ofst);
|
||||
|
||||
|
||||
|
||||
/// Creates an unweighted acceptor with a linear structure, with alternatives
|
||||
/// at each position. Epsilon is treated like a normal symbol here.
|
||||
/// Each position in "labels" must have at least one alternative.
|
||||
template<class Arc, class I>
|
||||
void MakeLinearAcceptorWithAlternatives(const std::vector<std::vector<I> > &labels,
|
||||
MutableFst<Arc> *ofst);
|
||||
|
||||
|
||||
/// Does PreDeterminize and DeterminizeStar and then removes the disambiguation symbols.
|
||||
/// This is a form of determinization that will never blow up.
|
||||
/// Note that ifst is non-const and can be considered to be destroyed by this
|
||||
/// operation.
|
||||
/// Does not do epsilon removal (RemoveEpsLocal)-- this is so it's safe to cast to
|
||||
/// log and do this, and maintain equivalence in tropical.
|
||||
|
||||
template<class Arc>
|
||||
void SafeDeterminizeWrapper(MutableFst<Arc> *ifst, MutableFst<Arc> *ofst, float delta = kDelta);
|
||||
|
||||
|
||||
/// SafeDeterminizeMinimizeWapper is as SafeDeterminizeWrapper except that it also
|
||||
/// minimizes (encoded minimization, which is safe). This algorithm will destroy "ifst".
|
||||
template<class Arc>
|
||||
void SafeDeterminizeMinimizeWrapper(MutableFst<Arc> *ifst, VectorFst<Arc> *ofst, float delta = kDelta);
|
||||
|
||||
|
||||
/// SafeDeterminizeMinimizeWapperInLog is as SafeDeterminizeMinimizeWrapper except
|
||||
/// it first casts tothe log semiring.
|
||||
void SafeDeterminizeMinimizeWrapperInLog(VectorFst<StdArc> *ifst, VectorFst<StdArc> *ofst, float delta = kDelta);
|
||||
|
||||
|
||||
|
||||
/// RemoveSomeInputSymbols removes any symbol that appears in "to_remove", from
|
||||
/// the input side of the FST, replacing them with epsilon.
|
||||
template<class Arc, class I>
|
||||
void RemoveSomeInputSymbols(const std::vector<I> &to_remove,
|
||||
MutableFst<Arc> *fst);
|
||||
|
||||
// MapInputSymbols will replace any input symbol i that is between 0 and
|
||||
// symbol_map.size()-1, with symbol_map[i]. It removes the input symbol
|
||||
// table of the FST.
|
||||
template<class Arc, class I>
|
||||
void MapInputSymbols(const std::vector<I> &symbol_map,
|
||||
MutableFst<Arc> *fst);
|
||||
|
||||
|
||||
template<class Arc>
|
||||
void RemoveWeights(MutableFst<Arc> *fst);
|
||||
|
||||
|
||||
|
||||
|
||||
/// Returns true if and only if the FST is such that the input symbols
|
||||
/// on arcs entering any given state all have the same value.
|
||||
/// if "start_is_epsilon", treat start-state as an epsilon input arc
|
||||
/// [i.e. ensure only epsilon can enter start-state].
|
||||
template<class Arc>
|
||||
bool PrecedingInputSymbolsAreSame(bool start_is_epsilon, const Fst<Arc> &fst);
|
||||
|
||||
|
||||
/// This is as PrecedingInputSymbolsAreSame, but with a functor f that maps labels to classes.
|
||||
/// The function tests whether the symbols preceding any given state are in the same
|
||||
/// class.
|
||||
/// Formally, f is of a type F that has an operator of type
|
||||
/// F::Result F::operator() (F::Arg a) const;
|
||||
/// where F::Result is an integer type and F::Arc can be constructed from Arc::Label.
|
||||
/// this must apply to valid labels and also to kNoLabel (so we can have a marker for
|
||||
/// the invalid labels.
|
||||
template<class Arc, class F>
|
||||
bool PrecedingInputSymbolsAreSameClass(bool start_is_epsilon, const Fst<Arc> &fst, const F &f);
|
||||
|
||||
|
||||
/// Returns true if and only if the FST is such that the input symbols
|
||||
/// on arcs exiting any given state all have the same value.
|
||||
/// If end_is_epsilon, treat end-state as an epsilon output arc [i.e. ensure
|
||||
/// end-states cannot have non-epsilon output transitions.]
|
||||
template<class Arc>
|
||||
bool FollowingInputSymbolsAreSame(bool end_is_epsilon, const Fst<Arc> &fst);
|
||||
|
||||
|
||||
template<class Arc, class F>
|
||||
bool FollowingInputSymbolsAreSameClass(bool end_is_epsilon, const Fst<Arc> &fst, const F &f);
|
||||
|
||||
|
||||
/// MakePrecedingInputSymbolsSame ensures that all arcs entering any given fst
|
||||
/// state have the same input symbol. It does this by detecting states
|
||||
/// that have differing input symbols going in, and inserting, for each of
|
||||
/// the preceding arcs with non-epsilon input symbol, a new dummy state that
|
||||
/// has an epsilon link to the fst state.
|
||||
/// If "start_is_epsilon", ensure that start-state can have only epsilon-links
|
||||
/// into it.
|
||||
template<class Arc>
|
||||
void MakePrecedingInputSymbolsSame(bool start_is_epsilon, MutableFst<Arc> *fst);
|
||||
|
||||
|
||||
/// As MakePrecedingInputSymbolsSame, but takes a functor object that maps labels to classes.
|
||||
template<class Arc, class F>
|
||||
void MakePrecedingInputSymbolsSameClass(bool start_is_epsilon, MutableFst<Arc> *fst, const F &f);
|
||||
|
||||
|
||||
/// MakeFollowingInputSymbolsSame ensures that all arcs exiting any given fst
|
||||
/// state have the same input symbol. It does this by detecting states that have
|
||||
/// differing input symbols on arcs that exit it, and inserting, for each of the
|
||||
/// following arcs with non-epsilon input symbol, a new dummy state that has an
|
||||
/// input-epsilon link from the fst state. The output symbol and weight stay on the
|
||||
/// link to the dummy state (in order to keep the FST output-deterministic and
|
||||
/// stochastic, if it already was).
|
||||
/// If end_is_epsilon, treat "being a final-state" like having an epsilon output
|
||||
/// link.
|
||||
template<class Arc>
|
||||
void MakeFollowingInputSymbolsSame(bool end_is_epsilon, MutableFst<Arc> *fst);
|
||||
|
||||
/// As MakeFollowingInputSymbolsSame, but takes a functor object that maps labels to classes.
|
||||
template<class Arc, class F>
|
||||
void MakeFollowingInputSymbolsSameClass(bool end_is_epsilon, MutableFst<Arc> *fst, const F &f);
|
||||
|
||||
|
||||
|
||||
|
||||
/// MakeLoopFst creates an FST that has a state that is both initial and
|
||||
/// final (weight == Weight::One()), and for each non-NULL pointer fsts[i],
|
||||
/// it has an arc out whose output-symbol is i and which goes to a
|
||||
/// sub-graph whose input language is equivalent to fsts[i], where the
|
||||
/// final-state becomes a transition to the loop-state. Each fst in "fsts"
|
||||
/// should be an acceptor. The fst MakeLoopFst returns is output-deterministic,
|
||||
/// but not output-epsilon free necessarily, and arcs are sorted on output label.
|
||||
/// Note: if some of the pointers in the input vector "fsts" have the same
|
||||
/// value, "MakeLoopFst" uses this to speed up the computation.
|
||||
|
||||
/// Formally: suppose I is the set of indexes i such that fsts[i] != NULL.
|
||||
/// Let L[i] be the language that the acceptor fsts[i] accepts.
|
||||
/// Let the language K be the set of input-output pairs i:l such
|
||||
/// that i in I and l in L[i]. Then the FST returned by MakeLoopFst
|
||||
/// accepts the language K*, where * is the Kleene closure (CLOSURE_STAR)
|
||||
/// of K.
|
||||
|
||||
/// We could have implemented this via a combination of "project",
|
||||
/// "concat", "union" and "closure". But that FST would have been
|
||||
/// less well optimized and would have a lot of final-states.
|
||||
|
||||
template<class Arc>
|
||||
VectorFst<Arc>* MakeLoopFst(const std::vector<const ExpandedFst<Arc> *> &fsts);
|
||||
|
||||
|
||||
/// ApplyProbabilityScale is applicable to FSTs in the log or tropical semiring.
|
||||
/// It multiplies the arc and final weights by "scale" [this is not the Mul
|
||||
/// operation of the semiring, it's actual multiplication, which is equivalent
|
||||
/// to taking a power in the semiring].
|
||||
template<class Arc>
|
||||
void ApplyProbabilityScale(float scale, MutableFst<Arc> *fst);
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/// EqualAlign is similar to RandGen, but it generates a sequence with exactly "length"
|
||||
/// input symbols. It returns true on success, false on failure (failure is partly
|
||||
/// random but should never happen in practice for normal speech models.)
|
||||
/// It generates a random path through the input FST, finds out which subset of the
|
||||
/// states it visits along the way have self-loops with inupt symbols on them, and
|
||||
/// outputs a path with exactly enough self-loops to have the requested number
|
||||
/// of input symbols.
|
||||
/// Note that EqualAlign does not use the probabilities on the FST. It just uses
|
||||
/// equal probabilities in the first stage of selection (since the output will anyway
|
||||
/// not be a truly random sample from the FST).
|
||||
/// The input fst "ifst" must be connected or this may enter an infinite loop.
|
||||
template<class Arc>
|
||||
bool EqualAlign(const Fst<Arc> &ifst, typename Arc::StateId length,
|
||||
int rand_seed, MutableFst<Arc> *ofst, int num_retries = 10);
|
||||
|
||||
|
||||
|
||||
// RemoveUselessArcs removes arcs such that there is no input symbol
|
||||
// sequence for which the best path through the FST would contain
|
||||
// those arcs [for these purposes, epsilon is not treated as a real symbol].
|
||||
// This is mainly geared towards decoding-graph FSTs which may contain
|
||||
// transitions that have less likely words on them that would never be
|
||||
// taken. We do not claim that this algorithm removes all such arcs;
|
||||
// it just does the best job it can.
|
||||
// Only works for tropical (not log) semiring as it uses
|
||||
// NaturalLess.
|
||||
template<class Arc>
|
||||
void RemoveUselessArcs(MutableFst<Arc> *fst);
|
||||
|
||||
|
||||
// PhiCompose is a version of composition where
|
||||
// the right hand FST (fst2) is treated as a backoff
|
||||
// LM, with the phi symbol (e.g. #0) treated as a
|
||||
// "failure transition", only taken when we don't
|
||||
// have a match for the requested symbol.
|
||||
template<class Arc>
|
||||
void PhiCompose(const Fst<Arc> &fst1,
|
||||
const Fst<Arc> &fst2,
|
||||
typename Arc::Label phi_label,
|
||||
MutableFst<Arc> *fst);
|
||||
|
||||
// PropagateFinal propagates final-probs through
|
||||
// "phi" transitions (note that here, phi_label may
|
||||
// be epsilon if you want). If you have a backoff LM
|
||||
// with special symbols ("phi") on the backoff arcs
|
||||
// instead of epsilon, you may use PhiCompose to compose
|
||||
// with it, but this won't do the right thing w.r.t.
|
||||
// final probabilities. You should first call PropagateFinal
|
||||
// on the FST with phi's i it (fst2 in PhiCompose above),
|
||||
// to fix this. If a state does not have a final-prob,
|
||||
// but has a phi transition, it makes the state's final-prob
|
||||
// (phi-prob * final-prob-of-dest-state), and does this
|
||||
// recursively i.e. follows phi transitions on the dest state
|
||||
// first. It behaves as if there were a super-final state
|
||||
// with a special symbol leading to it, from each currently
|
||||
// final state. Note that this may not behave as desired
|
||||
// if there are epsilons in your FST; it might be better
|
||||
// to remove those before calling this function.
|
||||
|
||||
template<class Arc>
|
||||
void PropagateFinal(typename Arc::Label phi_label,
|
||||
MutableFst<Arc> *fst);
|
||||
|
||||
// RhoCompose is a version of composition where
|
||||
// the right hand FST (fst2) has speciall "rho transitions"
|
||||
// which are taken whenever no normal transition matches; these
|
||||
// transitions will be rewritten with whatever symbol was on
|
||||
// the first FST.
|
||||
template<class Arc>
|
||||
void RhoCompose(const Fst<Arc> &fst1,
|
||||
const Fst<Arc> &fst2,
|
||||
typename Arc::Label rho_label,
|
||||
MutableFst<Arc> *fst);
|
||||
|
||||
/** This function returns true if, in the semiring of the FST, the sum (within
|
||||
the semiring) of all the arcs out of each state in the FST is one, to within
|
||||
delta. After MakeStochasticFst, this should be true (for a connected FST).
|
||||
|
||||
@param fst [in] the FST that we are testing.
|
||||
@param delta [in] the tolerance to within which we test equality to 1.
|
||||
@param min_sum [out] if non, NULL, contents will be set to the minimum sum of weights.
|
||||
@param max_sum [out] if non, NULL, contents will be set to the maximum sum of weights.
|
||||
@return Returns true if the FST is stochastic, and false otherwise.
|
||||
*/
|
||||
|
||||
template<class Arc>
|
||||
bool IsStochasticFst(const Fst<Arc> &fst,
|
||||
float delta = kDelta, // kDelta = 1.0/1024.0 by default.
|
||||
typename Arc::Weight *min_sum = NULL,
|
||||
typename Arc::Weight *max_sum = NULL);
|
||||
|
||||
|
||||
|
||||
|
||||
// IsStochasticFstInLog makes sure it's stochastic after casting to log.
|
||||
inline bool IsStochasticFstInLog(const Fst<StdArc> &fst,
|
||||
float delta = kDelta, // kDelta = 1.0/1024.0 by default.
|
||||
StdArc::Weight *min_sum = NULL,
|
||||
StdArc::Weight *max_sum = NULL);
|
||||
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
|
||||
#include "fstext/fstext-utils-inl.h"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,228 @@
|
||||
// fstext/grammar-context-fst.cc
|
||||
|
||||
// Copyright 2018 Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fstext/grammar-context-fst.h"
|
||||
#include "base/kaldi-error.h"
|
||||
#include "util/stl-utils.h"
|
||||
|
||||
namespace fst {
|
||||
using std::vector;
|
||||
|
||||
InverseLeftBiphoneContextFst::InverseLeftBiphoneContextFst(
|
||||
Label nonterm_phones_offset,
|
||||
const vector<int32>& phones,
|
||||
const vector<int32>& disambig_syms):
|
||||
nonterm_phones_offset_(nonterm_phones_offset),
|
||||
phone_syms_(phones),
|
||||
disambig_syms_(disambig_syms) {
|
||||
|
||||
{ // This block does some checks.
|
||||
std::vector<int32> all_inputs(phones);
|
||||
all_inputs.insert(all_inputs.end(), disambig_syms.begin(),
|
||||
disambig_syms.end());
|
||||
all_inputs.push_back(nonterm_phones_offset);
|
||||
size_t size = all_inputs.size();
|
||||
kaldi::SortAndUniq(&all_inputs);
|
||||
if (all_inputs.size() != size) {
|
||||
KALDI_ERR << "There was overlap between disambig symbols, phones, "
|
||||
"and/or --nonterm-phones-offset";
|
||||
}
|
||||
if (all_inputs.front() <= 0)
|
||||
KALDI_ERR << "Symbols <= 0 were passed in as phones, disambig-syms, "
|
||||
"or nonterm-phones-offset.";
|
||||
if (all_inputs.back() != nonterm_phones_offset) {
|
||||
// the value passed --nonterm-phones-offset is not higher numbered
|
||||
// than all the phones and disambig syms... do some more checking.
|
||||
for (int32 i = 1; i < 4; i++) {
|
||||
int32 symbol = nonterm_phones_offset + i;
|
||||
// None of the symbols --nonterm-phones-offset + {kNontermBos, kNontermBegin,
|
||||
// kNontermEnd, kNontermReenter, kNontermUserDefined}
|
||||
// (i.e. the special symbols plus the first user-defined symbol) may be
|
||||
// listed as phones or disambig symbols... this doesn't make sense. We
|
||||
// do allow disambig symbols to be higher-numbered than the nonterminal
|
||||
// sybols, just in case that happens to be needed, but they can't overlap.
|
||||
if (std::binary_search(all_inputs.begin(), all_inputs.end(), symbol)) {
|
||||
KALDI_ERR << "The symbol " << symbol
|
||||
<< " = --nonterm-phones-offset + " << i
|
||||
<< " was listed as a phone or disambig symbol.";
|
||||
}
|
||||
}
|
||||
}
|
||||
if (phone_syms_.empty())
|
||||
KALDI_WARN << "Context FST created but there are no phone symbols: probably "
|
||||
"input FST was empty.";
|
||||
}
|
||||
|
||||
// empty vector, will be the ilabel_info vector that corresponds to epsilon,
|
||||
// in case our FST needs to output epsilons.
|
||||
vector<int32> empty_vec;
|
||||
Label epsilon_label = FindLabel(empty_vec);
|
||||
// Make sure that a label is assigned for epsilon.
|
||||
KALDI_ASSERT(epsilon_label == 0);
|
||||
}
|
||||
|
||||
|
||||
InverseLeftBiphoneContextFst::Weight InverseLeftBiphoneContextFst::Final(StateId s) {
|
||||
if (s == 0 || phone_syms_.count(s) != 0 ||
|
||||
s == GetPhoneSymbolFor(kNontermEnd))
|
||||
return Weight::One();
|
||||
else
|
||||
return Weight::Zero();
|
||||
}
|
||||
|
||||
bool InverseLeftBiphoneContextFst::GetArc(
|
||||
StateId s, Label ilabel, Arc *arc) {
|
||||
// it's a rule of the DeterministicOnDemandFst that the ilabel cannot be zero.q
|
||||
KALDI_ASSERT(ilabel != 0);
|
||||
|
||||
arc->ilabel = ilabel;
|
||||
arc->weight = Weight::One();
|
||||
|
||||
if (s == 0 || phone_syms_.count(s) != 0) {
|
||||
// This is an epsilon or phone state.
|
||||
if (phone_syms_.count(ilabel) != 0) {
|
||||
// The ilabel is a phone.
|
||||
std::vector<int32> context_window(2);
|
||||
context_window[0] = s;
|
||||
context_window[1] = ilabel;
|
||||
arc->olabel = FindLabel(context_window);
|
||||
arc->nextstate = ilabel;
|
||||
return true;
|
||||
} else if (disambig_syms_.count(ilabel) != 0) {
|
||||
// the ilabel is a disambiguation symbol. Make a self-loop arc that
|
||||
// replicates the disambiguation symbol on the input.
|
||||
// The ilabel-info vector for disambig symbols is just a single element
|
||||
// consisting of the negative of the disambig symbols (for easier
|
||||
// identification from code).
|
||||
std::vector<int32> this_ilabel_info(1);
|
||||
this_ilabel_info[0] = -ilabel;
|
||||
arc->olabel = FindLabel(this_ilabel_info);
|
||||
arc->nextstate = s;
|
||||
return true;
|
||||
} else if (ilabel == GetPhoneSymbolFor(kNontermBegin) &&
|
||||
s == 0) {
|
||||
// We were at the start state and saw the symbol #nonterm_begin.
|
||||
// Output nothing, but transition to the special #nonterm_begin state.
|
||||
// when we're in that state, arcs for phones generate special
|
||||
// osymbols corresponding to pairs like (#nonterm_begin, p1).
|
||||
arc->olabel = 0;
|
||||
arc->nextstate = GetPhoneSymbolFor(kNontermBegin);
|
||||
return true;
|
||||
} else if (ilabel == GetPhoneSymbolFor(kNontermEnd)) {
|
||||
// we saw #nonterm_end.
|
||||
std::vector<int32> this_ilabel_info(2);
|
||||
this_ilabel_info[0] = -(GetPhoneSymbolFor(kNontermEnd));
|
||||
this_ilabel_info[1] = (s != 0 ? s : GetPhoneSymbolFor(kNontermBos));
|
||||
arc->olabel = FindLabel(this_ilabel_info);
|
||||
arc->nextstate = GetPhoneSymbolFor(kNontermEnd);
|
||||
return true;
|
||||
} else if (ilabel >= GetPhoneSymbolFor(kNontermUserDefined)) {
|
||||
// Assume this ilabel is a user-defined nonterminal.
|
||||
// Transition to the state kNontermUserDefined, with an olabel
|
||||
// (#nonterm:foo, p1) where 'p1' is the current left-context.
|
||||
std::vector<int32> this_ilabel_info(2);
|
||||
this_ilabel_info[0] = -ilabel;
|
||||
this_ilabel_info[1] = (s != 0 ? s : GetPhoneSymbolFor(kNontermBos));
|
||||
arc->olabel = FindLabel(this_ilabel_info);
|
||||
// the destination state is not specific to this user-defined symbol, it's
|
||||
// a generic destination state.
|
||||
arc->nextstate = GetPhoneSymbolFor(kNontermUserDefined);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else if (s == GetPhoneSymbolFor(kNontermBegin)) {
|
||||
if (phone_syms_.count(ilabel) != 0 || ilabel == GetPhoneSymbolFor(kNontermBos)) {
|
||||
std::vector<int32> this_ilabel_info(2);
|
||||
this_ilabel_info[0] = -GetPhoneSymbolFor(kNontermBegin);
|
||||
this_ilabel_info[1] = ilabel;
|
||||
arc->nextstate = (ilabel == GetPhoneSymbolFor(kNontermBos) ? 0 : ilabel);
|
||||
arc->olabel = FindLabel(this_ilabel_info);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else if (s == GetPhoneSymbolFor(kNontermEnd)) {
|
||||
return false;
|
||||
} else if (s == GetPhoneSymbolFor(kNontermUserDefined)) {
|
||||
if (phone_syms_.count(ilabel) != 0 || ilabel == GetPhoneSymbolFor(kNontermBos)) {
|
||||
std::vector<int32> this_ilabel_info(2);
|
||||
this_ilabel_info[0] = -GetPhoneSymbolFor(kNontermReenter);
|
||||
this_ilabel_info[1] = ilabel;
|
||||
arc->nextstate = (ilabel == GetPhoneSymbolFor(kNontermBos) ? 0 : ilabel);
|
||||
arc->olabel = FindLabel(this_ilabel_info);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
// likely code error.
|
||||
KALDI_ERR << "Invalid state encountered";
|
||||
return false; // won't get here. suppress compiler error.
|
||||
}
|
||||
}
|
||||
|
||||
StdArc::Label InverseLeftBiphoneContextFst::FindLabel(const vector<int32> &label_vec) {
|
||||
// Finds the ilabel corresponding to this vector (creates a new ilabel if
|
||||
// necessary).
|
||||
VectorToLabelMap::const_iterator iter = ilabel_map_.find(label_vec);
|
||||
if (iter == ilabel_map_.end()) { // Not already in map.
|
||||
Label this_label = ilabel_info_.size();
|
||||
ilabel_info_.push_back(label_vec);
|
||||
ilabel_map_[label_vec] = this_label;
|
||||
return this_label;
|
||||
} else {
|
||||
return iter->second;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void ComposeContextLeftBiphone(
|
||||
int32 nonterm_phones_offset,
|
||||
const vector<int32> &disambig_syms_in,
|
||||
const VectorFst<StdArc> &ifst,
|
||||
VectorFst<StdArc> *ofst,
|
||||
std::vector<std::vector<int32> > *ilabels) {
|
||||
|
||||
vector<int32> disambig_syms(disambig_syms_in);
|
||||
std::sort(disambig_syms.begin(), disambig_syms.end());
|
||||
|
||||
vector<int32> all_syms;
|
||||
GetInputSymbols(ifst, false/*no eps*/, &all_syms);
|
||||
std::sort(all_syms.begin(), all_syms.end());
|
||||
vector<int32> phones;
|
||||
for (size_t i = 0; i < all_syms.size(); i++)
|
||||
if (!std::binary_search(disambig_syms.begin(),
|
||||
disambig_syms.end(), all_syms[i]) &&
|
||||
all_syms[i] < nonterm_phones_offset)
|
||||
phones.push_back(all_syms[i]);
|
||||
|
||||
|
||||
InverseLeftBiphoneContextFst inv_c(nonterm_phones_offset,
|
||||
phones, disambig_syms);
|
||||
|
||||
// The following statement is equivalent to the following
|
||||
// (if FSTs had the '*' operator for composition):
|
||||
// (*ofst) = inv(inv_c) * (*ifst)
|
||||
ComposeDeterministicOnDemandInverse(ifst, &inv_c, ofst);
|
||||
|
||||
inv_c.SwapIlabelInfo(ilabels);
|
||||
}
|
||||
|
||||
} // end namespace fst
|
||||
@@ -0,0 +1,287 @@
|
||||
// fstext/grammar-context-fst.h
|
||||
|
||||
// Copyright 2018 Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
|
||||
#ifndef KALDI_FSTEXT_GRAMMAR_CONTEXT_FST_H_
|
||||
#define KALDI_FSTEXT_GRAMMAR_CONTEXT_FST_H_
|
||||
|
||||
/* This header defines a special form of the context FST "C" (the "C" in "HCLG")
|
||||
that integrates with our framework for building dynamic graphs for grammars
|
||||
that are too big to statically create, or graphs with on-the-fly pieces that
|
||||
you want to create at recognition time without building the whole graph.
|
||||
|
||||
This framework is limited to only work with models with left-biphone context.
|
||||
(Fortunately this doesn't impact results, as our best models are all 'chain'
|
||||
models with left biphone context).
|
||||
|
||||
The main code exported from here is the class InverseLeftBiphoneContextFst,
|
||||
which is similar to the InverseContextFst defined in context-fst.h, but
|
||||
is limited to left-biphone context and also supports certain special
|
||||
extensions we need to compile grammars.
|
||||
|
||||
See \ref grammar (../doc/grammar.dox) for high-level
|
||||
documentation on how this framework works.
|
||||
*/
|
||||
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
|
||||
#include "util/const-integer-set.h"
|
||||
#include "fstext/deterministic-fst.h"
|
||||
#include "fstext/context-fst.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
/**
|
||||
An anonymous enum to define some values for symbols used in our grammar-fst
|
||||
framework. Please understand this with reference to the documentation in
|
||||
\ref grammar (../doc/grammar.dox). This enum defines
|
||||
the values of nonterminal-related symbols in phones.txt. They are not
|
||||
the actual values-- they will be shifted by adding the value
|
||||
nonterm_phones_offset which is passed in by the command-line flag
|
||||
--nonterm-phones-offset.
|
||||
|
||||
*/
|
||||
|
||||
enum NonterminalValues {
|
||||
kNontermBos = 0, // #nonterm_bos
|
||||
kNontermBegin = 1, // #nonterm_begin
|
||||
kNontermEnd = 2, // #nonterm_end
|
||||
kNontermReenter = 3, // #nonterm_reenter
|
||||
kNontermUserDefined = 4, // the lowest-numbered user-defined nonterminal, e.g. #nonterm:foo
|
||||
// kNontermMediumNumber and kNontermBigNumber come into the encoding of
|
||||
// nonterminal-related symbols in HCLG.fst. The only hard constraint on them
|
||||
// is that kNontermBigNumber must be bigger than the biggest transition-id in
|
||||
// your system, and kNontermMediumNumber must be >0. These values were chosen
|
||||
// for ease of human inspection of numbers encoded with them.
|
||||
kNontermMediumNumber = 1000,
|
||||
kNontermBigNumber = 10000000
|
||||
};
|
||||
|
||||
|
||||
|
||||
// Returns the smallest multiple of 1000 that is strictly greater than
|
||||
// nonterm_phones_offset. Used in the encoding of special symbol in HCLG;
|
||||
// they are encoded as
|
||||
// special_symbol =
|
||||
// kNontermBigNumber + (nonterminal * encoding_multiple) + phone_index
|
||||
inline int32 GetEncodingMultiple(int32 nonterm_phones_offset) {
|
||||
int32 medium_number = static_cast<int32>(kNontermMediumNumber);
|
||||
return medium_number *
|
||||
((nonterm_phones_offset + medium_number) / medium_number);
|
||||
}
|
||||
|
||||
/**
|
||||
This is a variant of the function ComposeContext() which is to be used
|
||||
with our "grammar FST" framework (see \ref graph_context, i.e.
|
||||
../doc/grammar.dox, for more details). This does not take
|
||||
the 'context_width' and 'central_position' arguments because they are
|
||||
assumed to be 2 and 1 respectively (meaning, left-biphone phonetic context).
|
||||
|
||||
This function creates a context FST and composes it on the left with "ifst"
|
||||
to make "ofst".
|
||||
|
||||
@param [in] nonterm_phones_offset The integer id of the symbol
|
||||
#nonterm_bos in the phones.txt file. You can just set this
|
||||
to a large value (like 1 million) if you are not actually using
|
||||
nonterminals (e.g. for testing purposes).
|
||||
@param [in] disambig_syms List of disambiguation symbols, e.g. the integer
|
||||
ids of #0, #1, #2 ... in the phones.txt.
|
||||
@param [in,out] ifst The FST we are composing with C (e.g. LG.fst).
|
||||
@param [out] ofst Composed output FST (would be CLG.fst).
|
||||
@param [out] ilabels Vector, indexed by ilabel of CLG.fst, providing information
|
||||
about the meaning of that ilabel; see \ref tree_ilabel
|
||||
(http://kaldi-asr.org/doc/tree_externals.html#tree_ilabel)
|
||||
and also \ref grammar_special_clg
|
||||
(http://kaldi-asr.org/doc/grammar#grammar_special_clg).
|
||||
*/
|
||||
void ComposeContextLeftBiphone(
|
||||
int32 nonterm_phones_offset,
|
||||
const std::vector<int32> &disambig_syms,
|
||||
const VectorFst<StdArc> &ifst,
|
||||
VectorFst<StdArc> *ofst,
|
||||
std::vector<std::vector<int32> > *ilabels);
|
||||
|
||||
|
||||
|
||||
/*
|
||||
InverseLeftBiphoneContextFst represents the inverse of the context FST "C" (the "C" in
|
||||
"HCLG") which transduces from symbols representing phone context windows
|
||||
(e.g. "a, b, c") to individual phones, e.g. "a". So InverseContextFst
|
||||
transduces from phones to symbols representing phone context windows. The
|
||||
point is that the inverse is deterministic, so the DeterministicOnDemandFst
|
||||
interface is applicable, which turns out to be a convenient way to implement
|
||||
this.
|
||||
|
||||
This doesn't implement the full Fst interface, it implements the
|
||||
DeterministicOnDemandFst interface which is much simpler and which is
|
||||
sufficient for what we need to do with this.
|
||||
|
||||
Search for "hbka.pdf" ("Speech Recognition with Weighted Finite State
|
||||
Transducers") by M. Mohri, for more context.
|
||||
*/
|
||||
|
||||
class InverseLeftBiphoneContextFst: public DeterministicOnDemandFst<StdArc> {
|
||||
public:
|
||||
typedef StdArc Arc;
|
||||
typedef typename StdArc::StateId StateId;
|
||||
typedef typename StdArc::Weight Weight;
|
||||
typedef typename StdArc::Label Label;
|
||||
|
||||
/**
|
||||
Constructor. This does not take the arguments 'context_width' or
|
||||
'central_position' because they are assumed to be (2, 1) meaning a
|
||||
system with left-biphone context; and there is no subsequential
|
||||
symbol because it is not needed in systems without right context.
|
||||
|
||||
@param [in] nonterm_phones_offset The integer id of the symbol
|
||||
#nonterm_bos in the phones.txt file. You can just set this to
|
||||
a large value (like 1 million) if you are not actually using
|
||||
nonterminals (e.g. for testing purposes).
|
||||
@param [in] phones List of integer ids of phones, as you would see in phones.txt
|
||||
@param [in] disambig_syms List of integer ids of disambiguation symbols,
|
||||
e.g. the ids of #0, #1, #2 in phones.txt
|
||||
|
||||
See \ref graph_context for more details.
|
||||
*/
|
||||
InverseLeftBiphoneContextFst(Label nonterm_phones_offset,
|
||||
const std::vector<int32>& phones,
|
||||
const std::vector<int32>& disambig_syms);
|
||||
|
||||
/**
|
||||
Here is a note on the state space of InverseLeftBiphoneContextFst;
|
||||
see \ref grammar_special_c which has some documentation on this.
|
||||
|
||||
The state space uses the same numbering as phones.txt.
|
||||
|
||||
State 0 means the beginning-of-sequence state, where there is no left
|
||||
context.
|
||||
|
||||
For each phone p in the list 'phones' passed to the constructor (i.e. in
|
||||
the set passed to the constructor), the state 'p' corresponds to a
|
||||
left-context of that phone.
|
||||
|
||||
If p is equal to nonterm_phones_offset_ + kNontermBegin (i.e. the
|
||||
integer form of `\#nonterm_begin`), then this is the state we transition
|
||||
to when we see that symbol starting from left-context==0 (no context). The
|
||||
transition to this special state will have epsilon on the output. (talking
|
||||
here about inv(C), not C, so input/output are reversed).
|
||||
The state is nonfinal and when we see a regular phone p1 or #nonterm_bos, instead of
|
||||
outputting that phone in context, we output the pair (#nonterm_begin,p1) or
|
||||
(#nonterm_begin,#nonterm_bos). This state is not final.
|
||||
|
||||
If p is equal to nonterm_phones_offset_ + kNontermUserDefined, then this
|
||||
is the state we transition to when we see any user-defined nonterminal.
|
||||
Transitions to this special state have olabels of the form (#nonterm:foo,p1)
|
||||
where p1 is the preceding context (with #nonterm_begin if that context was
|
||||
0); transitions out of it have olabels of the form (#nonterm_reenter,p2), where
|
||||
p2 is the phone on the ilabel of that transition. Again: talking about inv(C).
|
||||
This state is not final.
|
||||
|
||||
If p is equal to nonterm_phones_offset_ + kNontermEnd, then this is
|
||||
the state we transition to when we see the ilabel #nonterm_end. The olabels
|
||||
on the transitions to it (talking here about inv(C), so ilabels and olabels
|
||||
are reversed) are of the form (#nonterm_end, p1) where p1 corresponds to the
|
||||
context we were in. This state is final.
|
||||
*/
|
||||
|
||||
|
||||
virtual StateId Start() { return 0; }
|
||||
|
||||
virtual Weight Final(StateId s);
|
||||
|
||||
/// Note: ilabel must not be epsilon.
|
||||
virtual bool GetArc(StateId s, Label ilabel, Arc *arc);
|
||||
|
||||
~InverseLeftBiphoneContextFst() { }
|
||||
|
||||
// Returns a reference to a vector<vector<int32> > with information about all
|
||||
// the input symbols of C (i.e. all the output symbols of this
|
||||
// InverseContextFst). See
|
||||
// "http://kaldi-asr.org/doc/tree_externals.html#tree_ilabel".
|
||||
const std::vector<std::vector<int32> > &IlabelInfo() const {
|
||||
return ilabel_info_;
|
||||
}
|
||||
|
||||
// A way to destructively obtain the ilabel-info. Only do this if you
|
||||
// are just about to destroy this object.
|
||||
void SwapIlabelInfo(std::vector<std::vector<int32> > *vec) { ilabel_info_.swap(*vec); }
|
||||
|
||||
private:
|
||||
|
||||
inline int32 GetPhoneSymbolFor(enum NonterminalValues n) {
|
||||
return nonterm_phones_offset_ + static_cast<int32>(n);
|
||||
}
|
||||
|
||||
/// Finds the label index corresponding to this context-window of phones
|
||||
/// (likely of width context_width_). Inserts it into the
|
||||
/// ilabel_info_/ilabel_map_ tables if necessary.
|
||||
Label FindLabel(const std::vector<int32> &label_info);
|
||||
|
||||
|
||||
// Map type to map from vectors of int32 (representing ilabel-info,
|
||||
// see http://kaldi-asr.org/doc/tree_externals.html#tree_ilabel) to
|
||||
// Label (the output label in this FST).
|
||||
typedef unordered_map<std::vector<int32>, Label,
|
||||
kaldi::VectorHasher<int32> > VectorToLabelMap;
|
||||
|
||||
|
||||
// The following three variables were also passed in by the caller:
|
||||
int32 nonterm_phones_offset_;
|
||||
|
||||
// 'phone_syms_' are a set of phone-ids, typically 1, 2, .. num_phones.
|
||||
kaldi::ConstIntegerSet<Label> phone_syms_;
|
||||
|
||||
// disambig_syms_ is the set of integer ids of the disambiguation symbols,
|
||||
// usually represented in text form as #0, #1, #2, etc. These are inserted
|
||||
// into the grammar (for #0) and the lexicon (for #1, #2, ...) in order to
|
||||
// make the composed FSTs determinizable. They are treated "specially" by the
|
||||
// context FST in that they are not part of the context, they are just "passed
|
||||
// through" via self-loops. See the Mohri chapter mrentioned above for more
|
||||
// information.
|
||||
kaldi::ConstIntegerSet<Label> disambig_syms_;
|
||||
|
||||
|
||||
// maps from vector<int32>, representing phonetic contexts of length
|
||||
// context_width_ - 1, to Label. These are actually the output labels of this
|
||||
// InverseContextFst (because of the "Inverse" part), but for historical
|
||||
// reasons and because we've used the term ilabels" in the documentation, we
|
||||
// still call these "ilabels").
|
||||
VectorToLabelMap ilabel_map_;
|
||||
|
||||
// ilabel_info_ is the reverse map of ilabel_map_.
|
||||
// Indexed by olabel (although we call this ilabel_info_ for historical
|
||||
// reasons and because is for the ilabels of C), ilabel_info_[i] gives
|
||||
// information about the meaning of each symbol on the input of C
|
||||
// aka the output of inv(C).
|
||||
// See "http://kaldi-asr.org/doc/tree_externals.html#tree_ilabel".
|
||||
std::vector<std::vector<int32> > ilabel_info_;
|
||||
|
||||
};
|
||||
|
||||
} // namespace fst
|
||||
|
||||
|
||||
#endif // KALDI_FSTEXT_GRAMMAR_CONTEXT_FST_H_
|
||||
@@ -0,0 +1,211 @@
|
||||
// fstext/kaldi-fst-io-inl.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2012-2015 Johns Hopkins University (Author: Daniel Povey)
|
||||
// 2013 Guoguo Chen
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_KALDI_FST_IO_INL_H_
|
||||
#define KALDI_FSTEXT_KALDI_FST_IO_INL_H_
|
||||
|
||||
#include "util/text-utils.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
template <class Arc>
|
||||
void WriteFstKaldi(std::ostream &os, bool binary,
|
||||
const VectorFst<Arc> &t) {
|
||||
bool ok;
|
||||
if (binary) {
|
||||
// Binary-mode writing.
|
||||
ok = t.Write(os, FstWriteOptions());
|
||||
} else {
|
||||
// Text-mode output. Note: we expect that t.InputSymbols() and
|
||||
// t.OutputSymbols() would always return NULL. The corresponding input
|
||||
// routine would not work if the FST actually had symbols attached. Write a
|
||||
// newline to start the FST; in a table, the first line of the FST will
|
||||
// appear on its own line.
|
||||
os << '\n';
|
||||
bool acceptor = false, write_one = false;
|
||||
FstPrinter<Arc> printer(t, t.InputSymbols(), t.OutputSymbols(),
|
||||
NULL, acceptor, write_one, "\t");
|
||||
printer.Print(&os, "<unknown>");
|
||||
if (os.fail())
|
||||
KALDI_ERR << "Stream failure detected writing FST to stream";
|
||||
// Write another newline as a terminating character. The read routine will
|
||||
// detect this [this is a Kaldi mechanism, not something in the original
|
||||
// OpenFst code].
|
||||
os << '\n';
|
||||
ok = os.good();
|
||||
}
|
||||
if (!ok) {
|
||||
KALDI_ERR << "Error writing FST to stream";
|
||||
}
|
||||
}
|
||||
|
||||
// Utility function used in ReadFstKaldi
|
||||
template <class W>
|
||||
inline bool StrToWeight(const std::string &s, bool allow_zero, W *w) {
|
||||
std::istringstream strm(s);
|
||||
strm >> *w;
|
||||
if (strm.fail() || (!allow_zero && *w == W::Zero())) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <class Arc>
|
||||
void ReadFstKaldi(std::istream &is, bool binary,
|
||||
VectorFst<Arc> *fst) {
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef typename Arc::StateId StateId;
|
||||
if (binary) {
|
||||
// We don't have access to the filename here, so write [unknown].
|
||||
VectorFst<Arc> *ans =
|
||||
VectorFst<Arc>::Read(is, fst::FstReadOptions(std::string("[unknown]")));
|
||||
if (ans == NULL) {
|
||||
KALDI_ERR << "Error reading FST from stream.";
|
||||
}
|
||||
*fst = *ans; // shallow copy.
|
||||
delete ans;
|
||||
} else {
|
||||
// Consume the \r on Windows, the \n that the text-form FST format starts
|
||||
// with, and any extra spaces that might have got in there somehow.
|
||||
while (std::isspace(is.peek()) && is.peek() != '\n') is.get();
|
||||
if (is.peek() == '\n') is.get(); // consume the newline.
|
||||
else { // saw spaces but no newline.. this is not expected.
|
||||
KALDI_ERR << "Reading FST: unexpected sequence of spaces "
|
||||
<< " at file position " << is.tellg();
|
||||
}
|
||||
using std::string;
|
||||
using std::vector;
|
||||
using kaldi::SplitStringToIntegers;
|
||||
using kaldi::ConvertStringToInteger;
|
||||
fst->DeleteStates();
|
||||
string line;
|
||||
size_t nline = 0;
|
||||
string separator = FLAGS_fst_field_separator + "\r\n";
|
||||
while (std::getline(is, line)) {
|
||||
nline++;
|
||||
vector<string> col;
|
||||
// on Windows we'll write in text and read in binary mode.
|
||||
kaldi::SplitStringToVector(line, separator.c_str(), true, &col);
|
||||
if (col.size() == 0) break; // Empty line is a signal to stop, in our
|
||||
// archive format.
|
||||
if (col.size() > 5) {
|
||||
KALDI_ERR << "Bad line in FST: " << line;
|
||||
}
|
||||
StateId s;
|
||||
if (!ConvertStringToInteger(col[0], &s)) {
|
||||
KALDI_ERR << "Bad line in FST: " << line;
|
||||
}
|
||||
while (s >= fst->NumStates())
|
||||
fst->AddState();
|
||||
if (nline == 1) fst->SetStart(s);
|
||||
|
||||
bool ok = true;
|
||||
Arc arc;
|
||||
Weight w;
|
||||
StateId d = s;
|
||||
switch (col.size()) {
|
||||
case 1:
|
||||
fst->SetFinal(s, Weight::One());
|
||||
break;
|
||||
case 2:
|
||||
if (!StrToWeight(col[1], true, &w)) ok = false;
|
||||
else fst->SetFinal(s, w);
|
||||
break;
|
||||
case 3: // 3 columns not ok for Lattice format; it's not an acceptor.
|
||||
ok = false;
|
||||
break;
|
||||
case 4:
|
||||
ok = ConvertStringToInteger(col[1], &arc.nextstate) &&
|
||||
ConvertStringToInteger(col[2], &arc.ilabel) &&
|
||||
ConvertStringToInteger(col[3], &arc.olabel);
|
||||
if (ok) {
|
||||
d = arc.nextstate;
|
||||
arc.weight = Weight::One();
|
||||
fst->AddArc(s, arc);
|
||||
}
|
||||
break;
|
||||
case 5:
|
||||
ok = ConvertStringToInteger(col[1], &arc.nextstate) &&
|
||||
ConvertStringToInteger(col[2], &arc.ilabel) &&
|
||||
ConvertStringToInteger(col[3], &arc.olabel) &&
|
||||
StrToWeight(col[4], false, &arc.weight);
|
||||
if (ok) {
|
||||
d = arc.nextstate;
|
||||
fst->AddArc(s, arc);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
ok = false;
|
||||
}
|
||||
while (d >= fst->NumStates()) fst->AddState();
|
||||
if (!ok)
|
||||
KALDI_ERR << "Bad line in FST: " << line;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
template<class Arc> // static
|
||||
bool VectorFstTplHolder<Arc>::Write(std::ostream &os, bool binary, const T &t) {
|
||||
try {
|
||||
WriteFstKaldi(os, binary, t);
|
||||
return true;
|
||||
} catch (...) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
template<class Arc> // static
|
||||
bool VectorFstTplHolder<Arc>::Read(std::istream &is) {
|
||||
Clear();
|
||||
int c = is.peek();
|
||||
if (c == -1) {
|
||||
KALDI_WARN << "End of stream detected reading Fst";
|
||||
return false;
|
||||
} else if (isspace(c)) { // The text form of the FST begins
|
||||
// with space (normally, '\n'), so this means it's text (the binary form
|
||||
// cannot begin with space because it starts with the FST Type() which is not
|
||||
// space).
|
||||
try {
|
||||
t_ = new VectorFst<Arc>();
|
||||
ReadFstKaldi(is, false, t_);
|
||||
} catch (...) {
|
||||
Clear();
|
||||
return false;
|
||||
}
|
||||
} else { // reading a binary FST.
|
||||
try {
|
||||
t_ = new VectorFst<Arc>();
|
||||
ReadFstKaldi(is, true, t_);
|
||||
} catch (...) {
|
||||
Clear();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace fst.
|
||||
|
||||
#endif // KALDI_FSTEXT_KALDI_FST_IO_INL_H_
|
||||
@@ -0,0 +1,145 @@
|
||||
// fstext/kaldi-fst-io.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2012-2015 Johns Hopkins University (Author: Daniel Povey)
|
||||
// 2013 Guoguo Chen
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fstext/kaldi-fst-io.h"
|
||||
#include "base/kaldi-error.h"
|
||||
#include "base/kaldi-math.h"
|
||||
#include "util/kaldi-io.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
VectorFst<StdArc> *ReadFstKaldi(std::string rxfilename) {
|
||||
if (rxfilename == "") rxfilename = "-"; // interpret "" as stdin,
|
||||
// for compatibility with OpenFst conventions.
|
||||
kaldi::Input ki(rxfilename);
|
||||
fst::FstHeader hdr;
|
||||
if (!hdr.Read(ki.Stream(), rxfilename))
|
||||
KALDI_ERR << "Reading FST: error reading FST header from "
|
||||
<< kaldi::PrintableRxfilename(rxfilename);
|
||||
FstReadOptions ropts("<unspecified>", &hdr);
|
||||
VectorFst<StdArc> *fst = VectorFst<StdArc>::Read(ki.Stream(), ropts);
|
||||
if (!fst)
|
||||
KALDI_ERR << "Could not read fst from "
|
||||
<< kaldi::PrintableRxfilename(rxfilename);
|
||||
return fst;
|
||||
}
|
||||
|
||||
// Register const fst to load it automatically. Other types like
|
||||
// olabel_lookahead or ngram or compact_fst should be registered
|
||||
// through OpenFst registration API.
|
||||
static fst::FstRegisterer<VectorFst<StdArc>> VectorFst_StdArc_registerer;
|
||||
static fst::FstRegisterer<ConstFst<StdArc>> ConstFst_StdArc_registerer;
|
||||
|
||||
Fst<StdArc> *ReadFstKaldiGeneric(std::string rxfilename, bool throw_on_err) {
|
||||
if (rxfilename == "") rxfilename = "-"; // interpret "" as stdin,
|
||||
// for compatibility with OpenFst conventions.
|
||||
kaldi::Input ki(rxfilename);
|
||||
fst::FstHeader hdr;
|
||||
// Read FstHeader which contains the type of FST
|
||||
if (!hdr.Read(ki.Stream(), rxfilename)) {
|
||||
if(throw_on_err) {
|
||||
KALDI_ERR << "Reading FST: error reading FST header from "
|
||||
<< kaldi::PrintableRxfilename(rxfilename);
|
||||
} else {
|
||||
KALDI_WARN << "We fail to read FST header from "
|
||||
<< kaldi::PrintableRxfilename(rxfilename)
|
||||
<< ". A NULL pointer is returned.";
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
// Check the type of Arc
|
||||
if (hdr.ArcType() != fst::StdArc::Type()) {
|
||||
if(throw_on_err) {
|
||||
KALDI_ERR << "FST with arc type " << hdr.ArcType() << " is not supported.";
|
||||
} else {
|
||||
KALDI_WARN << "Fst with arc type" << hdr.ArcType()
|
||||
<< " is not supported. A NULL pointer is returned.";
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
// Read the FST
|
||||
FstReadOptions ropts("<unspecified>", &hdr);
|
||||
Fst<StdArc> *fst = Fst<StdArc>::Read(ki.Stream(), ropts);
|
||||
if (!fst) {
|
||||
if(throw_on_err) {
|
||||
KALDI_ERR << "Could not read fst from "
|
||||
<< kaldi::PrintableRxfilename(rxfilename);
|
||||
} else {
|
||||
KALDI_WARN << "Could not read fst from "
|
||||
<< kaldi::PrintableRxfilename(rxfilename)
|
||||
<< ". A NULL pointer is returned.";
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
return fst;
|
||||
}
|
||||
|
||||
VectorFst<StdArc> *CastOrConvertToVectorFst(Fst<StdArc> *fst) {
|
||||
// This version currently supports ConstFst<StdArc> or VectorFst<StdArc>
|
||||
std::string real_type = fst->Type();
|
||||
KALDI_ASSERT(real_type == "vector" || real_type == "const");
|
||||
if (real_type == "vector") {
|
||||
return dynamic_cast<VectorFst<StdArc> *>(fst);
|
||||
} else {
|
||||
// As the 'fst' can't cast to VectorFst, we create a new
|
||||
// VectorFst<StdArc> initialized by 'fst', and delete 'fst'.
|
||||
VectorFst<StdArc> *new_fst = new VectorFst<StdArc>(*fst);
|
||||
delete fst;
|
||||
return new_fst;
|
||||
}
|
||||
}
|
||||
|
||||
void ReadFstKaldi(std::string rxfilename, fst::StdVectorFst *ofst) {
|
||||
fst::StdVectorFst *fst = ReadFstKaldi(rxfilename);
|
||||
*ofst = *fst;
|
||||
delete fst;
|
||||
}
|
||||
|
||||
void WriteFstKaldi(const VectorFst<StdArc> &fst,
|
||||
std::string wxfilename) {
|
||||
if (wxfilename == "") wxfilename = "-"; // interpret "" as stdout,
|
||||
// for compatibility with OpenFst conventions.
|
||||
bool write_binary = true, write_header = false;
|
||||
kaldi::Output ko(wxfilename, write_binary, write_header);
|
||||
FstWriteOptions wopts(kaldi::PrintableWxfilename(wxfilename));
|
||||
fst.Write(ko.Stream(), wopts);
|
||||
}
|
||||
|
||||
fst::VectorFst<fst::StdArc> *ReadAndPrepareLmFst(std::string rxfilename) {
|
||||
// ReadFstKaldi() will die with exception on failure.
|
||||
fst::VectorFst<fst::StdArc> *ans = fst::ReadFstKaldi(rxfilename);
|
||||
if (ans->Properties(fst::kAcceptor, true) == 0) {
|
||||
// If it's not already an acceptor, project on the output, i.e. copy olabels
|
||||
// to ilabels. Generally the G.fst's on disk will have the disambiguation
|
||||
// symbol #0 on the input symbols of the backoff arc, and projection will
|
||||
// replace them with epsilons which is what is on the output symbols of
|
||||
// those arcs.
|
||||
fst::Project(ans, fst::PROJECT_OUTPUT);
|
||||
}
|
||||
if (ans->Properties(fst::kILabelSorted, true) == 0) {
|
||||
// Make sure LM is sorted on ilabel.
|
||||
fst::ILabelCompare<fst::StdArc> ilabel_comp;
|
||||
fst::ArcSort(ans, ilabel_comp);
|
||||
}
|
||||
return ans;
|
||||
}
|
||||
|
||||
} // end namespace fst
|
||||
@@ -0,0 +1,158 @@
|
||||
// fstext/kaldi-fst-io.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2012-2015 Johns Hopkins University (Author: Daniel Povey)
|
||||
// 2013 Guoguo Chen
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_KALDI_FST_IO_H_
|
||||
#define KALDI_FSTEXT_KALDI_FST_IO_H_
|
||||
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
#include <fst/script/print-impl.h>
|
||||
#include "base/kaldi-common.h"
|
||||
|
||||
// Some functions for writing Fsts.
|
||||
// I/O for FSTs is a bit of a mess, and not very well integrated with Kaldi's
|
||||
// generic I/O mechanisms, because we want files containing just FSTs to
|
||||
// be readable by OpenFST's native binaries, which is not compatible
|
||||
// with the normal \0B header that identifies Kaldi files as containing
|
||||
// binary data.
|
||||
// So use the functions here with your eyes open, and with caution!
|
||||
namespace fst {
|
||||
|
||||
// Read a binary FST using Kaldi I/O mechanisms (pipes, etc.)
|
||||
// On error returns NULL. Only supports VectorFst and exists
|
||||
// mainly for backward code compabibility.
|
||||
VectorFst<StdArc> *ReadFstKaldi(std::string rxfilename);
|
||||
|
||||
// Read a binary FST using Kaldi I/O mechanisms (pipes, etc.)
|
||||
// If it can't read the FST, if throw_on_err == true it throws using KALDI_ERR;
|
||||
// otherwise it prints a warning and returns. Note:this
|
||||
// doesn't support the text-mode option that we generally like to support.
|
||||
// This version currently supports ConstFst<StdArc> or VectorFst<StdArc>
|
||||
// (const-fst can give better performance for decoding). Other
|
||||
// types could be also loaded if registered inside OpenFst.
|
||||
Fst<StdArc> *ReadFstKaldiGeneric(std::string rxfilename,
|
||||
bool throw_on_err = true);
|
||||
|
||||
// This function attempts to dynamic_cast the pointer 'fst' (which will likely
|
||||
// have been returned by ReadFstGeneric()), to the more derived
|
||||
// type VectorFst<StdArc>. If this succeeds, it returns the same pointer;
|
||||
// if it fails, it converts the FST type (by creating a new VectorFst<stdArc>
|
||||
// initialized by 'fst'), prints a warning, and deletes 'fst'.
|
||||
VectorFst<StdArc> *CastOrConvertToVectorFst(Fst<StdArc> *fst);
|
||||
|
||||
// Version of ReadFstKaldi() that writes to a pointer. Assumes
|
||||
// the FST is binary with no binary marker. Crashes on error.
|
||||
void ReadFstKaldi(std::string rxfilename, VectorFst<StdArc> *ofst);
|
||||
|
||||
// Write an FST using Kaldi I/O mechanisms (pipes, etc.)
|
||||
// On error, throws using KALDI_ERR. For use only in code in fstbin/,
|
||||
// as it doesn't support the text-mode option.
|
||||
void WriteFstKaldi(const VectorFst<StdArc> &fst,
|
||||
std::string wxfilename);
|
||||
|
||||
// This is a more general Kaldi-type-IO mechanism of writing FSTs to
|
||||
// streams, supporting binary or text-mode writing. (note: we just
|
||||
// write the integers, symbol tables are not supported).
|
||||
// On error, throws using KALDI_ERR.
|
||||
template <class Arc>
|
||||
void WriteFstKaldi(std::ostream &os, bool binary,
|
||||
const VectorFst<Arc> &fst);
|
||||
|
||||
// A generic Kaldi-type-IO mechanism of reading FSTs from streams,
|
||||
// supporting binary or text-mode reading/writing.
|
||||
template <class Arc>
|
||||
void ReadFstKaldi(std::istream &is, bool binary,
|
||||
VectorFst<Arc> *fst);
|
||||
|
||||
// Read an FST file for LM (G.fst) and make it an acceptor,
|
||||
// and make sure it is sorted on labels
|
||||
fst::VectorFst<fst::StdArc> *ReadAndPrepareLmFst(std::string rxfilename);
|
||||
|
||||
// This is a Holder class with T = VectorFst<Arc>, that meets the requirements
|
||||
// of a Holder class as described in ../util/kaldi-holder.h. This enables us to
|
||||
// read/write collections of FSTs indexed by strings, using the Table concept (
|
||||
// see ../util/kaldi-table.h).
|
||||
// Originally it was only templated on T = VectorFst<StdArc>, but as the keyword
|
||||
// spotting stuff introduced more types of FSTs, we made it also templated on
|
||||
// the arc.
|
||||
template<class Arc>
|
||||
class VectorFstTplHolder {
|
||||
public:
|
||||
typedef VectorFst<Arc> T;
|
||||
|
||||
VectorFstTplHolder(): t_(NULL) { }
|
||||
|
||||
static bool Write(std::ostream &os, bool binary, const T &t);
|
||||
|
||||
void Copy(const T &t) { // copies it into the holder.
|
||||
Clear();
|
||||
t_ = new T(t);
|
||||
}
|
||||
|
||||
// Reads into the holder.
|
||||
bool Read(std::istream &is);
|
||||
|
||||
// It's potentially a binary format, so must read in binary mode (linefeed
|
||||
// translation will corrupt the file. We don't know till we open the file if
|
||||
// it's really binary, so we need to read in binary mode to be on the safe
|
||||
// side. Extra linefeeds won't matter, the text-mode reading code ignores
|
||||
// them.
|
||||
static bool IsReadInBinary() { return true; }
|
||||
|
||||
T &Value() {
|
||||
// code error if !t_.
|
||||
if (!t_) KALDI_ERR << "VectorFstTplHolder::Value() called wrongly.";
|
||||
return *t_;
|
||||
}
|
||||
|
||||
void Clear() {
|
||||
if (t_) {
|
||||
delete t_;
|
||||
t_ = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
void Swap(VectorFstTplHolder<Arc> *other) {
|
||||
std::swap(t_, other->t_);
|
||||
}
|
||||
|
||||
bool ExtractRange(const VectorFstTplHolder<Arc> &other,
|
||||
const std::string &range) {
|
||||
KALDI_ERR << "ExtractRange is not defined for this type of holder.";
|
||||
return false;
|
||||
}
|
||||
|
||||
~VectorFstTplHolder() { Clear(); }
|
||||
// No destructor. Assignment and
|
||||
// copy constructor take their default implementations.
|
||||
private:
|
||||
KALDI_DISALLOW_COPY_AND_ASSIGN(VectorFstTplHolder);
|
||||
T *t_;
|
||||
};
|
||||
|
||||
// Now make the original VectorFstHolder as the typedef of VectorFstHolder<StdArc>.
|
||||
typedef VectorFstTplHolder<StdArc> VectorFstHolder;
|
||||
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
#include "fstext/kaldi-fst-io-inl.h"
|
||||
#endif
|
||||
@@ -0,0 +1,282 @@
|
||||
// fstext/lattice-utils-inl.h
|
||||
|
||||
// Copyright 2009-2012 Microsoft Corporation Johns Hopkins University (Author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_LATTICE_UTILS_INL_H_
|
||||
#define KALDI_FSTEXT_LATTICE_UTILS_INL_H_
|
||||
// Do not include this file directly. It is included by lattice-utils.h
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
/* Convert from FST with arc-type Weight, to one with arc-type
|
||||
CompactLatticeWeight. Uses FactorFst to identify chains
|
||||
of states which can be turned into a single output arc. */
|
||||
|
||||
template<class Weight, class Int>
|
||||
void ConvertLattice(
|
||||
const ExpandedFst<ArcTpl<Weight> > &ifst,
|
||||
MutableFst<ArcTpl<CompactLatticeWeightTpl<Weight, Int> > > *ofst,
|
||||
bool invert) {
|
||||
typedef ArcTpl<Weight> Arc;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef CompactLatticeWeightTpl<Weight, Int> CompactWeight;
|
||||
typedef ArcTpl<CompactWeight> CompactArc;
|
||||
|
||||
VectorFst<ArcTpl<Weight> > ffst;
|
||||
std::vector<std::vector<Int> > labels;
|
||||
if (invert) // normal case: want the ilabels as sequences on the arcs of
|
||||
Factor(ifst, &ffst, &labels); // the output... Factor makes seqs of
|
||||
// ilabels.
|
||||
else {
|
||||
VectorFst<ArcTpl<Weight> > invfst(ifst);
|
||||
Invert(&invfst);
|
||||
Factor(invfst, &ffst, &labels);
|
||||
}
|
||||
|
||||
TopSort(&ffst); // Put the states in ffst in topological order, which is
|
||||
// easier on the eye when reading the text-form lattices and corresponds to
|
||||
// what we get when we generate the lattices in the decoder.
|
||||
|
||||
ofst->DeleteStates();
|
||||
|
||||
// The states will be numbered exactly the same as the original FST.
|
||||
// Add the states to the new FST.
|
||||
StateId num_states = ffst.NumStates();
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
StateId news = ofst->AddState();
|
||||
assert(news == s);
|
||||
}
|
||||
ofst->SetStart(ffst.Start());
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
Weight final_weight = ffst.Final(s);
|
||||
if (final_weight != Weight::Zero()) {
|
||||
CompactWeight final_compact_weight(final_weight, std::vector<Int>());
|
||||
ofst->SetFinal(s, final_compact_weight);
|
||||
}
|
||||
for (ArcIterator<ExpandedFst<Arc> > iter(ffst, s);
|
||||
!iter.Done();
|
||||
iter.Next()) {
|
||||
const Arc &arc = iter.Value();
|
||||
KALDI_PARANOID_ASSERT(arc.weight != Weight::Zero());
|
||||
// note: zero-weight arcs not allowed anyway so weight should not be zero,
|
||||
// but no harm in checking.
|
||||
CompactArc compact_arc(arc.olabel, arc.olabel,
|
||||
CompactWeight(arc.weight, labels[arc.ilabel]),
|
||||
arc.nextstate);
|
||||
ofst->AddArc(s, compact_arc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<class Weight, class Int>
|
||||
void ConvertLattice(
|
||||
const ExpandedFst<ArcTpl<CompactLatticeWeightTpl<Weight, Int> > > &ifst,
|
||||
MutableFst<ArcTpl<Weight> > *ofst,
|
||||
bool invert) {
|
||||
typedef ArcTpl<Weight> Arc;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Label Label;
|
||||
typedef CompactLatticeWeightTpl<Weight, Int> CompactWeight;
|
||||
typedef ArcTpl<CompactWeight> CompactArc;
|
||||
ofst->DeleteStates();
|
||||
// make the states in the new FST have the same numbers as
|
||||
// the original ones, and add chains of states as necessary
|
||||
// to encode the string-valued weights.
|
||||
StateId num_states = ifst.NumStates();
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
StateId news = ofst->AddState();
|
||||
assert(news == s);
|
||||
}
|
||||
ofst->SetStart(ifst.Start());
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
CompactWeight final_weight = ifst.Final(s);
|
||||
if (final_weight != CompactWeight::Zero()) {
|
||||
StateId cur_state = s;
|
||||
size_t string_length = final_weight.String().size();
|
||||
for (size_t n = 0; n < string_length; n++) {
|
||||
StateId next_state = ofst->AddState();
|
||||
Label ilabel = 0;
|
||||
Arc arc(ilabel, final_weight.String()[n],
|
||||
(n == 0 ? final_weight.Weight() : Weight::One()),
|
||||
next_state);
|
||||
if (invert) std::swap(arc.ilabel, arc.olabel);
|
||||
ofst->AddArc(cur_state, arc);
|
||||
cur_state = next_state;
|
||||
}
|
||||
ofst->SetFinal(cur_state,
|
||||
string_length > 0 ? Weight::One() : final_weight.Weight());
|
||||
}
|
||||
for (ArcIterator<ExpandedFst<CompactArc> > iter(ifst, s);
|
||||
!iter.Done();
|
||||
iter.Next()) {
|
||||
const CompactArc &arc = iter.Value();
|
||||
size_t string_length = arc.weight.String().size();
|
||||
StateId cur_state = s;
|
||||
// for all but the last element in the string--
|
||||
// add a temporary state.
|
||||
for (size_t n = 0 ; n+1 < string_length; n++) {
|
||||
StateId next_state = ofst->AddState();
|
||||
Label ilabel = (n == 0 ? arc.ilabel : 0),
|
||||
olabel = static_cast<Label>(arc.weight.String()[n]);
|
||||
Weight weight = (n == 0 ? arc.weight.Weight() : Weight::One());
|
||||
Arc new_arc(ilabel, olabel, weight, next_state);
|
||||
if (invert) std::swap(new_arc.ilabel, new_arc.olabel);
|
||||
ofst->AddArc(cur_state, new_arc);
|
||||
cur_state = next_state;
|
||||
}
|
||||
Label ilabel = (string_length <= 1 ? arc.ilabel : 0),
|
||||
olabel = (string_length > 0 ? arc.weight.String()[string_length-1] : 0);
|
||||
Weight weight = (string_length <= 1 ? arc.weight.Weight() : Weight::One());
|
||||
Arc new_arc(ilabel, olabel, weight, arc.nextstate);
|
||||
if (invert) std::swap(new_arc.ilabel, new_arc.olabel);
|
||||
ofst->AddArc(cur_state, new_arc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This function converts lattices between float and double;
|
||||
// it works for both CompactLatticeWeight and LatticeWeight.
|
||||
template<class WeightIn, class WeightOut>
|
||||
void ConvertLattice(
|
||||
const ExpandedFst<ArcTpl<WeightIn> > &ifst,
|
||||
MutableFst<ArcTpl<WeightOut> > *ofst) {
|
||||
typedef ArcTpl<WeightIn> ArcIn;
|
||||
typedef ArcTpl<WeightOut> ArcOut;
|
||||
typedef typename ArcIn::StateId StateId;
|
||||
ofst->DeleteStates();
|
||||
// The states will be numbered exactly the same as the original FST.
|
||||
// Add the states to the new FST.
|
||||
StateId num_states = ifst.NumStates();
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
StateId news = ofst->AddState();
|
||||
assert(news == s);
|
||||
}
|
||||
ofst->SetStart(ifst.Start());
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
WeightIn final_iweight = ifst.Final(s);
|
||||
if (final_iweight != WeightIn::Zero()) {
|
||||
WeightOut final_oweight;
|
||||
ConvertLatticeWeight(final_iweight, &final_oweight);
|
||||
ofst->SetFinal(s, final_oweight);
|
||||
}
|
||||
for (ArcIterator<ExpandedFst<ArcIn> > iter(ifst, s);
|
||||
!iter.Done();
|
||||
iter.Next()) {
|
||||
ArcIn arc = iter.Value();
|
||||
KALDI_PARANOID_ASSERT(arc.weight != WeightIn::Zero());
|
||||
ArcOut oarc;
|
||||
ConvertLatticeWeight(arc.weight, &oarc.weight);
|
||||
oarc.ilabel = arc.ilabel;
|
||||
oarc.olabel = arc.olabel;
|
||||
oarc.nextstate = arc.nextstate;
|
||||
ofst->AddArc(s, oarc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
template<class Weight, class ScaleFloat>
|
||||
void ScaleLattice(
|
||||
const std::vector<std::vector<ScaleFloat> > &scale,
|
||||
MutableFst<ArcTpl<Weight> > *fst) {
|
||||
assert(scale.size() == 2 && scale[0].size() == 2 && scale[1].size() == 2);
|
||||
if (scale == DefaultLatticeScale()) // nothing to do.
|
||||
return;
|
||||
typedef ArcTpl<Weight> Arc;
|
||||
typedef MutableFst<Arc> Fst;
|
||||
typedef typename Arc::StateId StateId;
|
||||
StateId num_states = fst->NumStates();
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
for (MutableArcIterator<Fst> aiter(fst, s);
|
||||
!aiter.Done();
|
||||
aiter.Next()) {
|
||||
Arc arc = aiter.Value();
|
||||
arc.weight = Weight(ScaleTupleWeight(arc.weight, scale));
|
||||
aiter.SetValue(arc);
|
||||
}
|
||||
Weight final_weight = fst->Final(s);
|
||||
if (final_weight != Weight::Zero())
|
||||
fst->SetFinal(s, Weight(ScaleTupleWeight(final_weight, scale)));
|
||||
}
|
||||
}
|
||||
|
||||
template<class Weight, class Int>
|
||||
void RemoveAlignmentsFromCompactLattice(
|
||||
MutableFst<ArcTpl<CompactLatticeWeightTpl<Weight, Int> > > *fst) {
|
||||
typedef CompactLatticeWeightTpl<Weight, Int> W;
|
||||
typedef ArcTpl<W> Arc;
|
||||
typedef MutableFst<Arc> Fst;
|
||||
typedef typename Arc::StateId StateId;
|
||||
StateId num_states = fst->NumStates();
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
for (MutableArcIterator<Fst> aiter(fst, s);
|
||||
!aiter.Done();
|
||||
aiter.Next()) {
|
||||
Arc arc = aiter.Value();
|
||||
arc.weight = W(arc.weight.Weight(), std::vector<Int>());
|
||||
aiter.SetValue(arc);
|
||||
}
|
||||
W final_weight = fst->Final(s);
|
||||
if (final_weight != W::Zero())
|
||||
fst->SetFinal(s, W(final_weight.Weight(), std::vector<Int>()));
|
||||
}
|
||||
}
|
||||
|
||||
template<class Weight, class Int>
|
||||
bool CompactLatticeHasAlignment(
|
||||
const ExpandedFst<ArcTpl<CompactLatticeWeightTpl<Weight, Int> > > &fst) {
|
||||
typedef CompactLatticeWeightTpl<Weight, Int> W;
|
||||
typedef ArcTpl<W> Arc;
|
||||
typedef ExpandedFst<Arc> Fst;
|
||||
typedef typename Arc::StateId StateId;
|
||||
StateId num_states = fst.NumStates();
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
for (ArcIterator<Fst> aiter(fst, s);
|
||||
!aiter.Done();
|
||||
aiter.Next()) {
|
||||
const Arc &arc = aiter.Value();
|
||||
if (!arc.weight.String().empty()) return true;
|
||||
}
|
||||
W final_weight = fst.Final(s);
|
||||
if (!final_weight.String().empty()) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
template <class Real>
|
||||
void ConvertFstToLattice(
|
||||
const ExpandedFst<ArcTpl<TropicalWeight> > &ifst,
|
||||
MutableFst<ArcTpl<LatticeWeightTpl<Real> > > *ofst) {
|
||||
int32 num_states_cache = 50000;
|
||||
fst::CacheOptions cache_opts(true, num_states_cache);
|
||||
fst::MapFstOptions mapfst_opts(cache_opts);
|
||||
StdToLatticeMapper<Real> mapper;
|
||||
MapFst<StdArc, ArcTpl<LatticeWeightTpl<Real> >,
|
||||
StdToLatticeMapper<Real> > map_fst(ifst, mapper, mapfst_opts);
|
||||
*ofst = map_fst;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,331 @@
|
||||
// fstext/lattice-utils-test.cc
|
||||
|
||||
// Copyright 2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fstext/lattice-utils.h"
|
||||
#include "fstext/fst-test-utils.h"
|
||||
#include "base/kaldi-math.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
template<class Weight, class Int> void TestConvert(bool invert) {
|
||||
typedef ArcTpl<Weight> Arc;
|
||||
typedef ArcTpl<CompactLatticeWeightTpl<Weight, Int> > CompactArc;
|
||||
for(int i = 0; i < 5; i++) {
|
||||
VectorFst<Arc> *fst = RandFst<Arc>();
|
||||
std::cout << "FST before converting to compact-arc is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
VectorFst<CompactArc> ofst;
|
||||
ConvertLattice<Weight, Int>(*fst, &ofst, invert);
|
||||
|
||||
std::cout << "FST after converting is:\n";
|
||||
{
|
||||
FstPrinter<CompactArc> fstprinter(ofst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
VectorFst<Arc> origfst;
|
||||
ConvertLattice<Weight, Int>(ofst, &origfst, invert);
|
||||
std::cout << "FST after back conversion is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(origfst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
assert(RandEquivalent(*fst, origfst, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
delete fst;
|
||||
}
|
||||
}
|
||||
|
||||
// This tests the ShortestPath algorithm, and by proxy, tests the
|
||||
// NaturalLess template etc.
|
||||
|
||||
template<class Weight, class Int> void TestShortestPath() {
|
||||
for (int p = 0; p < 10; p++) {
|
||||
typedef ArcTpl<Weight> Arc;
|
||||
typedef ArcTpl<CompactLatticeWeightTpl<Weight, Int> > CompactArc;
|
||||
for(int i = 0; i < 5; i++) {
|
||||
VectorFst<Arc> *fst = RandPairFst<Arc>();
|
||||
std::cout << "Testing shortest path\n";
|
||||
std::cout << "FST before converting to compact-arc is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
VectorFst<CompactArc> cfst;
|
||||
ConvertLattice<Weight, Int>(*fst, &cfst, false); // invert == false
|
||||
|
||||
|
||||
{
|
||||
VectorFst<Arc> nbest_fst_1;
|
||||
ShortestPath(*fst, &nbest_fst_1, 1);
|
||||
VectorFst<Arc> nbest_fst_2;
|
||||
ShortestPath(*fst, &nbest_fst_2, 3);
|
||||
VectorFst<Arc> nbest_fst_1b;
|
||||
ShortestPath(nbest_fst_2, &nbest_fst_1b, 1);
|
||||
|
||||
|
||||
assert(ApproxEqual(ShortestDistance(nbest_fst_1),
|
||||
ShortestDistance(nbest_fst_1b)));
|
||||
|
||||
// since semiring is idempotent, this should succeed too.
|
||||
assert(ApproxEqual(ShortestDistance(*fst),
|
||||
ShortestDistance(nbest_fst_1b)));
|
||||
}
|
||||
{
|
||||
VectorFst<CompactArc> nbest_fst_1;
|
||||
ShortestPath(cfst, &nbest_fst_1, 1);
|
||||
VectorFst<CompactArc> nbest_fst_2;
|
||||
ShortestPath(cfst, &nbest_fst_2, 3);
|
||||
VectorFst<CompactArc> nbest_fst_1b;
|
||||
ShortestPath(nbest_fst_2, &nbest_fst_1b, 1);
|
||||
|
||||
assert(ApproxEqual(ShortestDistance(nbest_fst_1),
|
||||
ShortestDistance(nbest_fst_1b)));
|
||||
// since semiring is idempotent, this should succeed too.
|
||||
assert(ApproxEqual(ShortestDistance(cfst),
|
||||
ShortestDistance(nbest_fst_1b)));
|
||||
}
|
||||
|
||||
delete fst;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
template<class Int> void TestConvert2() {
|
||||
typedef ArcTpl<LatticeWeightTpl<float> > ArcF;
|
||||
typedef ArcTpl<LatticeWeightTpl<double> > ArcD;
|
||||
typedef ArcTpl<CompactLatticeWeightTpl<LatticeWeightTpl<float>, Int> > CArcF;
|
||||
typedef ArcTpl<CompactLatticeWeightTpl<LatticeWeightTpl<double>, Int> > CArcD;
|
||||
|
||||
for(int i = 0; i < 2; i++) {
|
||||
{
|
||||
VectorFst<ArcF> *fst1 = RandPairFst<ArcF>();
|
||||
VectorFst<ArcD> fst2;
|
||||
VectorFst<ArcF> fst3;
|
||||
ConvertLattice(*fst1, &fst2);
|
||||
ConvertLattice(fst2, &fst3);
|
||||
|
||||
assert(RandEquivalent(*fst1, fst3, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
delete fst1;
|
||||
}
|
||||
|
||||
{
|
||||
VectorFst<ArcF> *fst1 = RandPairFst<ArcF>();
|
||||
VectorFst<CArcF> cfst1, cfst3;
|
||||
ConvertLattice(*fst1, &cfst1);
|
||||
VectorFst<CArcD> cfst2;
|
||||
ConvertLattice(cfst1, &cfst2);
|
||||
ConvertLattice(cfst2, &cfst3);
|
||||
assert(RandEquivalent(cfst1, cfst3, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
delete fst1;
|
||||
}
|
||||
|
||||
{
|
||||
VectorFst<ArcF> *fst1 = RandPairFst<ArcF>();
|
||||
VectorFst<CArcD> cfst1, cfst3;
|
||||
ConvertLattice(*fst1, &cfst1);
|
||||
VectorFst<CArcF> cfst2;
|
||||
ConvertLattice(cfst1, &cfst2);
|
||||
ConvertLattice(cfst2, &cfst3);
|
||||
assert(RandEquivalent(cfst1, cfst3, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
delete fst1;
|
||||
}
|
||||
|
||||
{
|
||||
VectorFst<ArcD> *fst1 = RandPairFst<ArcD>();
|
||||
VectorFst<CArcD> cfst1, cfst3;
|
||||
ConvertLattice(*fst1, &cfst1);
|
||||
VectorFst<CArcF> cfst2;
|
||||
ConvertLattice(cfst1, &cfst2);
|
||||
ConvertLattice(cfst2, &cfst3);
|
||||
assert(RandEquivalent(cfst1, cfst3, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
delete fst1;
|
||||
}
|
||||
|
||||
{
|
||||
VectorFst<ArcD> *fst1 = RandPairFst<ArcD>();
|
||||
VectorFst<CArcF> cfst1;
|
||||
ConvertLattice(*fst1, &cfst1);
|
||||
VectorFst<ArcD> fst2;
|
||||
ConvertLattice(cfst1, &fst2);
|
||||
assert(RandEquivalent(*fst1, fst2, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
delete fst1;
|
||||
}
|
||||
|
||||
{
|
||||
VectorFst<ArcF> *fst1 = RandPairFst<ArcF>();
|
||||
VectorFst<CArcD> cfst1;
|
||||
ConvertLattice(*fst1, &cfst1);
|
||||
VectorFst<ArcF> fst2;
|
||||
ConvertLattice(cfst1, &fst2);
|
||||
assert(RandEquivalent(*fst1, fst2, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
delete fst1;
|
||||
}
|
||||
|
||||
{
|
||||
VectorFst<ArcD> *fst1 = RandPairFst<ArcD>();
|
||||
VectorFst<CArcF> cfst1;
|
||||
ConvertLattice(*fst1, &cfst1);
|
||||
VectorFst<ArcD> fst2;
|
||||
ConvertLattice(cfst1, &fst2);
|
||||
assert(RandEquivalent(*fst1, fst2, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
delete fst1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// use TestConvertPair when the Weight can be constructed from
|
||||
// a pair of floats.
|
||||
template<class Weight, class Int> void TestConvertPair(bool invert) {
|
||||
typedef ArcTpl<Weight> Arc;
|
||||
typedef ArcTpl<CompactLatticeWeightTpl<Weight, Int> > CompactArc;
|
||||
for(int i = 0; i < 2; i++) {
|
||||
VectorFst<Arc> *fst = RandPairFst<Arc>();
|
||||
/*std::cout << "FST before converting to compact-arc is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, NULL, NULL, NULL, false, true);
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}*/
|
||||
VectorFst<CompactArc> ofst;
|
||||
ConvertLattice<Weight, Int>(*fst, &ofst, invert);
|
||||
|
||||
/*std::cout << "FST after converting is:\n";
|
||||
{
|
||||
FstPrinter<CompactArc> fstprinter(ofst, NULL, NULL, NULL, false, true);
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}*/
|
||||
VectorFst<Arc> origfst;
|
||||
ConvertLattice<Weight, Int>(ofst, &origfst, invert);
|
||||
/*std::cout << "FST after back conversion is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(origfst, NULL, NULL, NULL, false, true);
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}*/
|
||||
|
||||
assert(RandEquivalent(*fst, origfst, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
delete fst;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// use TestConvertPair when the Weight can be constructed from
|
||||
// a pair of floats.
|
||||
template<class Weight, class Int> void TestScalePair(bool invert) {
|
||||
std::vector<std::vector<double> > scale1 = DefaultLatticeScale(),
|
||||
scale2 = DefaultLatticeScale();
|
||||
// important that all these numbers exactly representable as floats..
|
||||
// exact floating-point comparisons are used in LatticeWeight, and
|
||||
// this exactness is being tested here.. this test will fail for
|
||||
// other types of number.
|
||||
if (kaldi::Rand() % 4 == 0) {
|
||||
scale1[0][0] = 2.0;
|
||||
scale2[0][0] = 0.5;
|
||||
scale1[1][1] = 4.0;
|
||||
scale2[1][1] = 0.25;
|
||||
} else if (kaldi::Rand() % 3 == 0) {
|
||||
// use that [1 0.25; 0 1] [ 1 -0.25; 0 1] is the unit matrix.
|
||||
scale1[0][1] = 0.25;
|
||||
scale2[0][1] = -0.25;
|
||||
} else if (kaldi::Rand() % 2 == 0) {
|
||||
scale1[1][0] = 0.25;
|
||||
scale2[1][0] = -0.25;
|
||||
}
|
||||
|
||||
|
||||
typedef ArcTpl<Weight> Arc;
|
||||
typedef ArcTpl<CompactLatticeWeightTpl<Weight, Int> > CompactArc;
|
||||
for(int i = 0; i < 2; i++) {
|
||||
VectorFst<Arc> *fst = RandPairFst<Arc>();
|
||||
/*std::cout << "FST before converting to compact-arc is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, NULL, NULL, NULL, false, true);
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}*/
|
||||
VectorFst<CompactArc> ofst;
|
||||
ConvertLattice<Weight, Int>(*fst, &ofst, invert);
|
||||
ScaleLattice(scale1, &ofst);
|
||||
/*std::cout << "FST after converting and scaling is:\n";
|
||||
{
|
||||
FstPrinter<CompactArc> fstprinter(ofst, NULL, NULL, NULL, false, true);
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}*/
|
||||
VectorFst<Arc> origfst;
|
||||
ConvertLattice<Weight, Int>(ofst, &origfst, invert);
|
||||
ScaleLattice(scale2, &origfst);
|
||||
/*std::cout << "FST after back conversion and scaling is:\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(origfst, NULL, NULL, NULL, false, true);
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}*/
|
||||
// If RandEquivalent doesn't work, it could be due to a nasty issue related to the use
|
||||
// of exact floating-point comparisons in the Plus function of LatticeWeight.
|
||||
if (!RandEquivalent(*fst, origfst, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/)) {
|
||||
std::cerr << "Warn, randequivalent returned false. Checking equivalence another way.\n";
|
||||
assert(Equal(*fst, origfst));
|
||||
}
|
||||
delete fst;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
int main() {
|
||||
using namespace fst;
|
||||
|
||||
typedef ::int64 int64;
|
||||
typedef ::uint64 uint64;
|
||||
typedef ::int32 int32;
|
||||
typedef ::uint32 uint32;
|
||||
|
||||
{
|
||||
typedef LatticeWeightTpl<float> LatticeWeight;
|
||||
for(int i = 0; i < 2; i++) {
|
||||
bool invert = (i % 2);
|
||||
TestConvert<TropicalWeight, int32>(invert);
|
||||
TestConvertPair<LatticeWeight, int32>(invert);
|
||||
TestConvertPair<LatticeWeight, size_t>(invert);
|
||||
TestConvertPair<LexicographicWeight<TropicalWeight, TropicalWeight>, size_t>(invert);
|
||||
TestScalePair<LatticeWeight, int32>(invert);
|
||||
TestScalePair<LatticeWeight, size_t>(invert);
|
||||
TestScalePair<LexicographicWeight<TropicalWeight, TropicalWeight>, size_t>(invert);
|
||||
}
|
||||
}
|
||||
{
|
||||
typedef LatticeWeightTpl<double> LatticeWeight;
|
||||
TestShortestPath<LatticeWeight, int32>();
|
||||
TestConvert2<int32>();
|
||||
for(int i = 0; i < 2; i++) {
|
||||
bool invert = (i % 2);
|
||||
TestConvertPair<LatticeWeight, int32>(invert);
|
||||
TestConvertPair<LatticeWeight, size_t>(invert);
|
||||
TestConvertPair<LexicographicWeight<TropicalWeight, TropicalWeight>, size_t>(invert);
|
||||
TestScalePair<LatticeWeight, int32>(invert);
|
||||
TestScalePair<LatticeWeight, size_t>(invert);
|
||||
TestScalePair<LexicographicWeight<TropicalWeight, TropicalWeight>, size_t>(invert);
|
||||
}
|
||||
}
|
||||
std::cout << "Tests succeeded\n";
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
// fstext/lattice-utils.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#ifndef KALDI_FSTEXT_LATTICE_UTILS_H_
|
||||
#define KALDI_FSTEXT_LATTICE_UTILS_H_
|
||||
|
||||
#include "fst/fstlib.h"
|
||||
#include "fstext/lattice-weight.h"
|
||||
#include "fstext/factor.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
// The template ConvertLattice does conversions to and from
|
||||
// LatticeWeight FSTs and CompactLatticeWeight FSTs, and
|
||||
// between float and double, and to convert from LatticeWeight
|
||||
// to TropicalWeight. It's used in the I/O code for lattices,
|
||||
// and for converting lattices to standard FSTs (e.g. for creating
|
||||
// decoding graphs from lattices).
|
||||
|
||||
|
||||
/**
|
||||
Convert lattice from a normal FST to a CompactLattice FST.
|
||||
This is a bit like converting to the Gallic semiring, except
|
||||
the semiring behaves in a different way (designed to take
|
||||
the best path).
|
||||
Note: the ilabels end up as the symbols on the arcs of the
|
||||
output acceptor, and the olabels go to the strings. To make
|
||||
it the other way around (useful for the speech-recognition
|
||||
application), set invert=true [the default].
|
||||
*/
|
||||
template<class Weight, class Int>
|
||||
void ConvertLattice(
|
||||
const ExpandedFst<ArcTpl<Weight> > &ifst,
|
||||
MutableFst<ArcTpl<CompactLatticeWeightTpl<Weight, Int> > > *ofst,
|
||||
bool invert = true);
|
||||
|
||||
/**
|
||||
Convert lattice CompactLattice format to Lattice. This is a bit
|
||||
like converting from the Gallic semiring. As for any CompactLattice, "ifst"
|
||||
must be an acceptor (i.e., ilabels and olabels should be identical). If
|
||||
invert=false, the labels on "ifst" become the ilabels on "ofst" and the
|
||||
strings in the weights of "ifst" becomes the olabels. If invert=true
|
||||
[default], this is reversed (useful for speech recognition lattices; our
|
||||
standard non-compact format has the words on the output side to match HCLG).
|
||||
*/
|
||||
template<class Weight, class Int>
|
||||
void ConvertLattice(
|
||||
const ExpandedFst<ArcTpl<CompactLatticeWeightTpl<Weight, Int> > > &ifst,
|
||||
MutableFst<ArcTpl<Weight> > *ofst,
|
||||
bool invert = true);
|
||||
|
||||
|
||||
/**
|
||||
Convert between CompactLattices and Lattices of different floating point types...
|
||||
this works between any pair of weight types for which ConvertLatticeWeight
|
||||
is defined (c.f. lattice-weight.h), and also includes conversion from
|
||||
LatticeWeight to TropicalWeight.
|
||||
*/
|
||||
template<class WeightIn, class WeightOut>
|
||||
void ConvertLattice(
|
||||
const ExpandedFst<ArcTpl<WeightIn> > &ifst,
|
||||
MutableFst<ArcTpl<WeightOut> > *ofst);
|
||||
|
||||
|
||||
// Now define some ConvertLattice functions that require two phases of conversion (don't
|
||||
// bother coding these separately as they will be used rarely.
|
||||
|
||||
// Lattice with float to CompactLattice with double.
|
||||
template<class Int>
|
||||
void ConvertLattice(const ExpandedFst<ArcTpl<LatticeWeightTpl<float> > > &ifst,
|
||||
MutableFst<ArcTpl<CompactLatticeWeightTpl<LatticeWeightTpl<double>, Int> > > *ofst) {
|
||||
VectorFst<ArcTpl<CompactLatticeWeightTpl<LatticeWeightTpl<float>, Int> > > fst;
|
||||
ConvertLattice(ifst, &fst);
|
||||
ConvertLattice(fst, ofst);
|
||||
}
|
||||
|
||||
// Lattice with double to CompactLattice with float.
|
||||
template<class Int>
|
||||
void ConvertLattice(const ExpandedFst<ArcTpl<LatticeWeightTpl<double> > > &ifst,
|
||||
MutableFst<ArcTpl<CompactLatticeWeightTpl<LatticeWeightTpl<float>, Int> > > *ofst) {
|
||||
VectorFst<ArcTpl<CompactLatticeWeightTpl<LatticeWeightTpl<double>, Int> > > fst;
|
||||
ConvertLattice(ifst, &fst);
|
||||
ConvertLattice(fst, ofst);
|
||||
}
|
||||
|
||||
/// Converts CompactLattice with double to Lattice with float.
|
||||
template<class Int>
|
||||
void ConvertLattice(const ExpandedFst<ArcTpl<CompactLatticeWeightTpl<LatticeWeightTpl<double>, Int> > > &ifst,
|
||||
MutableFst<ArcTpl<LatticeWeightTpl<float> > > *ofst) {
|
||||
VectorFst<ArcTpl<CompactLatticeWeightTpl<LatticeWeightTpl<float>, Int> > > fst;
|
||||
ConvertLattice(ifst, &fst);
|
||||
ConvertLattice(fst, ofst);
|
||||
}
|
||||
|
||||
/// Converts CompactLattice with float to Lattice with double.
|
||||
template<class Int>
|
||||
void ConvertLattice(const ExpandedFst<ArcTpl<CompactLatticeWeightTpl<LatticeWeightTpl<float>, Int> > > &ifst,
|
||||
MutableFst<ArcTpl<LatticeWeightTpl<double> > > *ofst) {
|
||||
VectorFst<ArcTpl<CompactLatticeWeightTpl<LatticeWeightTpl<double>, Int> > > fst;
|
||||
ConvertLattice(ifst, &fst);
|
||||
ConvertLattice(fst, ofst);
|
||||
}
|
||||
|
||||
/// Converts TropicalWeight to LatticeWeight (puts all the weight on
|
||||
/// the first float in the lattice's pair).
|
||||
template <class Real>
|
||||
void ConvertFstToLattice(
|
||||
const ExpandedFst<ArcTpl<TropicalWeight> > &ifst,
|
||||
MutableFst<ArcTpl<LatticeWeightTpl<Real> > > *ofst);
|
||||
|
||||
|
||||
/** Returns a default 2x2 matrix scaling factor for LatticeWeight */
|
||||
inline std::vector<std::vector<double> > DefaultLatticeScale() {
|
||||
std::vector<std::vector<double> > ans(2);
|
||||
ans[0].resize(2, 0.0);
|
||||
ans[1].resize(2, 0.0);
|
||||
ans[0][0] = ans[1][1] = 1.0;
|
||||
return ans;
|
||||
}
|
||||
|
||||
inline std::vector<std::vector<double> > AcousticLatticeScale(double acwt) {
|
||||
std::vector<std::vector<double> > ans(2);
|
||||
ans[0].resize(2, 0.0);
|
||||
ans[1].resize(2, 0.0);
|
||||
ans[0][0] = 1.0;
|
||||
ans[1][1] = acwt;
|
||||
return ans;
|
||||
}
|
||||
|
||||
inline std::vector<std::vector<double> > GraphLatticeScale(double lmwt) {
|
||||
std::vector<std::vector<double> > ans(2);
|
||||
ans[0].resize(2, 0.0);
|
||||
ans[1].resize(2, 0.0);
|
||||
ans[0][0] = lmwt;
|
||||
ans[1][1] = 1.0;
|
||||
return ans;
|
||||
}
|
||||
|
||||
inline std::vector<std::vector<double> > LatticeScale(double lmwt, double acwt) {
|
||||
std::vector<std::vector<double> > ans(2);
|
||||
ans[0].resize(2, 0.0);
|
||||
ans[1].resize(2, 0.0);
|
||||
ans[0][0] = lmwt;
|
||||
ans[1][1] = acwt;
|
||||
return ans;
|
||||
}
|
||||
|
||||
|
||||
/** Scales the pairs of weights in LatticeWeight or CompactLatticeWeight by
|
||||
viewing the pair (a, b) as a 2-vector and pre-multiplying by the 2x2 matrix
|
||||
in "scale". E.g. typically scale would equal
|
||||
[ 1 0;
|
||||
0 acwt ]
|
||||
if we want to scale the acoustics by "acwt".
|
||||
*/
|
||||
template<class Weight, class ScaleFloat>
|
||||
void ScaleLattice(
|
||||
const std::vector<std::vector<ScaleFloat> > &scale,
|
||||
MutableFst<ArcTpl<Weight> > *fst);
|
||||
|
||||
/// Removes state-level alignments (the strings that are
|
||||
/// part of the weights).
|
||||
template<class Weight, class Int>
|
||||
void RemoveAlignmentsFromCompactLattice(
|
||||
MutableFst<ArcTpl<CompactLatticeWeightTpl<Weight, Int> > > *fst);
|
||||
|
||||
/// Returns true if lattice has alignments, i.e. it has
|
||||
/// any nonempty strings inside its weights.
|
||||
template<class Weight, class Int>
|
||||
bool CompactLatticeHasAlignment(
|
||||
const ExpandedFst<ArcTpl<CompactLatticeWeightTpl<Weight, Int> > > &fst);
|
||||
|
||||
|
||||
/// Class StdToLatticeMapper maps a normal arc (StdArc)
|
||||
/// to a LatticeArc by putting the StdArc weight as the first
|
||||
/// element of the LatticeWeight. Useful when doing LM
|
||||
/// rescoring.
|
||||
template<class Real>
|
||||
class StdToLatticeMapper {
|
||||
typedef LatticeWeightTpl<Real> LatticeWeight;
|
||||
typedef ArcTpl<LatticeWeight> LatticeArc;
|
||||
public:
|
||||
LatticeArc operator()(const StdArc &arc) {
|
||||
// Note: we have to check whether the arc's weight is zero below,
|
||||
// and if so return (infinity, infinity) and not (infinity, zero),
|
||||
// because (infinity, zero) is not a valid LatticeWeight, which should
|
||||
// either be both finite, or both infinite (i.e. Zero()).
|
||||
return LatticeArc(arc.ilabel, arc.olabel,
|
||||
LatticeWeight(arc.weight.Value(),
|
||||
arc.weight == StdArc::Weight::Zero() ?
|
||||
arc.weight.Value() : 0.0),
|
||||
arc.nextstate);
|
||||
}
|
||||
MapFinalAction FinalAction() { return MAP_NO_SUPERFINAL; }
|
||||
|
||||
MapSymbolsAction InputSymbolsAction() { return MAP_COPY_SYMBOLS; }
|
||||
|
||||
MapSymbolsAction OutputSymbolsAction() { return MAP_COPY_SYMBOLS; }
|
||||
|
||||
// I believe all properties are preserved.
|
||||
uint64 Properties(uint64 props) { return props; }
|
||||
};
|
||||
|
||||
|
||||
/// Class LatticeToStdMapper maps a LatticeArc to a normal arc (StdArc)
|
||||
/// by adding the elements of the LatticeArc weight.
|
||||
|
||||
template<class Real>
|
||||
class LatticeToStdMapper {
|
||||
typedef LatticeWeightTpl<Real> LatticeWeight;
|
||||
typedef ArcTpl<LatticeWeight> LatticeArc;
|
||||
public:
|
||||
StdArc operator()(const LatticeArc &arc) {
|
||||
return StdArc(arc.ilabel, arc.olabel,
|
||||
StdArc::Weight(arc.weight.Value1() + arc.weight.Value2()),
|
||||
arc.nextstate);
|
||||
}
|
||||
MapFinalAction FinalAction() { return MAP_NO_SUPERFINAL; }
|
||||
|
||||
MapSymbolsAction InputSymbolsAction() { return MAP_COPY_SYMBOLS; }
|
||||
|
||||
MapSymbolsAction OutputSymbolsAction() { return MAP_COPY_SYMBOLS; }
|
||||
|
||||
// I believe all properties are preserved.
|
||||
uint64 Properties(uint64 props) { return props; }
|
||||
};
|
||||
|
||||
|
||||
template<class Weight, class Int>
|
||||
void PruneCompactLattice(
|
||||
Weight beam,
|
||||
MutableFst<ArcTpl<CompactLatticeWeightTpl<Weight, Int> > > *fst);
|
||||
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
#include "fstext/lattice-utils-inl.h"
|
||||
|
||||
#endif // KALDI_FSTEXT_LATTICE_UTILS_H_
|
||||
@@ -0,0 +1,197 @@
|
||||
// fstext/lattice-weight-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "base/kaldi-math.h"
|
||||
#include "fstext/lattice-weight.h"
|
||||
|
||||
namespace fst {
|
||||
using std::vector;
|
||||
using std::cout;
|
||||
// these typedefs are the same as in ../lat/kaldi-lattice.h, but
|
||||
// just used here for testing (doesn't matter if they get out of
|
||||
// sync).
|
||||
typedef float BaseFloat;
|
||||
|
||||
typedef LatticeWeightTpl<BaseFloat> LatticeWeight;
|
||||
|
||||
typedef CompactLatticeWeightTpl<LatticeWeight, int32> CompactLatticeWeight;
|
||||
|
||||
typedef CompactLatticeWeightCommonDivisorTpl<LatticeWeight, int32>
|
||||
CompactLatticeWeightCommonDivisor;
|
||||
|
||||
|
||||
LatticeWeight RandomLatticeWeight() {
|
||||
int tmp = kaldi::Rand() % 4;
|
||||
if (tmp == 0) {
|
||||
return LatticeWeight::Zero();
|
||||
} else if (tmp == 1) {
|
||||
return LatticeWeight( 1, 2); // sometimes return special values..
|
||||
} else if (tmp == 2) {
|
||||
return LatticeWeight( 2, 1); // this tests more thoroughly certain properties...
|
||||
} else {
|
||||
return LatticeWeight( 100 * kaldi::RandGauss(), 100 * kaldi::RandGauss());
|
||||
}
|
||||
}
|
||||
|
||||
CompactLatticeWeight RandomCompactLatticeWeight() {
|
||||
LatticeWeight w = RandomLatticeWeight();
|
||||
if (w == LatticeWeight::Zero()) {
|
||||
return CompactLatticeWeight(w, vector<int32>());
|
||||
} else {
|
||||
int32 len = kaldi::Rand() % 4;
|
||||
vector<int32> str;
|
||||
for(int32 i = 0; i < len; i++)
|
||||
str.push_back(kaldi::Rand() % 10 + 1);
|
||||
return CompactLatticeWeight(w, str);
|
||||
}
|
||||
}
|
||||
|
||||
void LatticeWeightTest() {
|
||||
for(int32 i = 0; i < 100; i++) {
|
||||
LatticeWeight l1 = RandomLatticeWeight(), l2 = RandomLatticeWeight();
|
||||
LatticeWeight l3 = Plus(l1, l2);
|
||||
LatticeWeight l4 = Times(l1, l2);
|
||||
BaseFloat f1 = l1.Value1() + l1.Value2(), f2 = l2.Value1() + l2.Value2(), f3 = l3.Value1() + l3.Value2(),
|
||||
f4 = l4.Value1() + l4.Value2();
|
||||
kaldi::AssertEqual(std::min(f1, f2), f3);
|
||||
kaldi::AssertEqual(f1 + f2, f4);
|
||||
|
||||
KALDI_ASSERT(Plus(l3, l3) == l3);
|
||||
KALDI_ASSERT(Plus(l1, l2) == Plus(l2, l1)); // commutativity of plus
|
||||
KALDI_ASSERT(Times(l1, l2) == Times(l2, l1)); // commutativity of Times (true for this semiring, not always)
|
||||
KALDI_ASSERT(Plus(l3, LatticeWeight::Zero()) == l3); // x + 0 = x
|
||||
KALDI_ASSERT(Times(l3, LatticeWeight::One()) == l3); // x * 1 = x
|
||||
KALDI_ASSERT(Times(l3, LatticeWeight::Zero()) == LatticeWeight::Zero()); // x * 0 = 0
|
||||
|
||||
KALDI_ASSERT(l3.Reverse().Reverse() == l3);
|
||||
|
||||
NaturalLess<LatticeWeight> nl;
|
||||
bool a = nl(l1, l2);
|
||||
bool b = (Plus(l1, l2) == l1 && l1 != l2);
|
||||
KALDI_ASSERT(a == b);
|
||||
|
||||
KALDI_ASSERT(Compare(l1, Plus(l1, l2)) != 1); // so do not have l1 > l1 + l2
|
||||
LatticeWeight l5 = RandomLatticeWeight(), l6 = RandomLatticeWeight();
|
||||
{
|
||||
LatticeWeight wa = Times(Plus(l1, l2), Plus(l5, l6)),
|
||||
wb = Plus(Times(l1, l5), Plus(Times(l1, l6),
|
||||
Plus(Times(l2, l5), Times(l2, l6))));
|
||||
if (!ApproxEqual(wa, wb)) {
|
||||
std::cout << "l1 = " << l1 << ", l2 = " << l2
|
||||
<< ", l5 = " << l5 << ", l6 = " << l6 << "\n";
|
||||
std::cout << "ERROR: " << wa << " != " << wb << "\n";
|
||||
}
|
||||
// KALDI_ASSERT(Times(Plus(l1, l2), Plus(l5, l6))
|
||||
// == Plus(Times(l1, l5), Plus(Times(l1,l6),
|
||||
// Plus(Times(l2, l5), Times(l2, l6))))); // * distributes over +
|
||||
}
|
||||
KALDI_ASSERT(l1.Member() && l2.Member() && l3.Member() && l4.Member()
|
||||
&& l5.Member() && l6.Member());
|
||||
if (l2 != LatticeWeight::Zero())
|
||||
KALDI_ASSERT(ApproxEqual(Divide(Times(l1, l2), l2), l1)); // (a*b) / b = a if b != 0
|
||||
KALDI_ASSERT(ApproxEqual(l1, l1.Quantize()));
|
||||
|
||||
std::ostringstream s1;
|
||||
s1 << l1;
|
||||
std::istringstream s2(s1.str());
|
||||
s2 >> l2;
|
||||
KALDI_ASSERT(ApproxEqual(l1, l2, 0.001));
|
||||
std::cout << s1.str() << '\n';
|
||||
{
|
||||
std::ostringstream s1b;
|
||||
l1.Write(s1b);
|
||||
std::istringstream s2b(s1b.str());
|
||||
l3.Read(s2b);
|
||||
KALDI_ASSERT(l1 == l3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void CompactLatticeWeightTest() {
|
||||
for(int32 i = 0; i < 100; i++) {
|
||||
CompactLatticeWeight l1 = RandomCompactLatticeWeight(), l2 = RandomCompactLatticeWeight();
|
||||
CompactLatticeWeight l3 = Plus(l1, l2);
|
||||
CompactLatticeWeight l4 = Times(l1, l2);
|
||||
|
||||
KALDI_ASSERT(Plus(l3, l3) == l3);
|
||||
KALDI_ASSERT(Plus(l1, l2) == Plus(l2, l1)); // commutativity of plus
|
||||
KALDI_ASSERT(Plus(l3, CompactLatticeWeight::Zero()) == l3); // x + 0 = x
|
||||
KALDI_ASSERT(Times(l3, CompactLatticeWeight::One()) == l3); // x * 1 = x
|
||||
KALDI_ASSERT(Times(l3, CompactLatticeWeight::Zero()) == CompactLatticeWeight::Zero()); // x * 0 = 0
|
||||
NaturalLess<CompactLatticeWeight> nl;
|
||||
bool a = nl(l1, l2);
|
||||
bool b = (Plus(l1, l2) == l1 && l1 != l2);
|
||||
KALDI_ASSERT(a == b);
|
||||
|
||||
KALDI_ASSERT(Compare(l1, Plus(l1, l2)) != 1); // so do not have l1 > l1 + l2
|
||||
CompactLatticeWeight l5 = RandomCompactLatticeWeight(), l6 = RandomCompactLatticeWeight();
|
||||
KALDI_ASSERT(Times(Plus(l1, l2), Plus(l5, l6)) ==
|
||||
Plus(Times(l1, l5), Plus(Times(l1, l6),
|
||||
Plus(Times(l2, l5), Times(l2, l6))))); // * distributes over +
|
||||
KALDI_ASSERT(l1.Member() && l2.Member() && l3.Member() && l4.Member()
|
||||
&& l5.Member() && l6.Member());
|
||||
if (l2 != CompactLatticeWeight::Zero()) {
|
||||
KALDI_ASSERT(ApproxEqual(Divide(Times(l1, l2), l2, DIVIDE_RIGHT), l1)); // (a*b) / b = a if b != 0
|
||||
KALDI_ASSERT(ApproxEqual(Divide(Times(l2, l1), l2, DIVIDE_LEFT), l1)); // (a*b) / b = a if b != 0
|
||||
}
|
||||
KALDI_ASSERT(ApproxEqual(l1, l1.Quantize()));
|
||||
|
||||
std::ostringstream s1;
|
||||
s1 << l1;
|
||||
std::istringstream s2(s1.str());
|
||||
s2 >> l2;
|
||||
KALDI_ASSERT(ApproxEqual(l1, l2));
|
||||
std::cout << s1.str() << '\n';
|
||||
|
||||
{
|
||||
std::ostringstream s1b;
|
||||
l1.Write(s1b);
|
||||
std::istringstream s2b(s1b.str());
|
||||
l3.Read(s2b);
|
||||
KALDI_ASSERT(l1 == l3);
|
||||
}
|
||||
|
||||
CompactLatticeWeightCommonDivisor divisor;
|
||||
std::cout << "l5 = " << l5 << '\n';
|
||||
std::cout << "l6 = " << l6 << '\n';
|
||||
l1 = divisor(l5, l6);
|
||||
std::cout << "div = " << l1 << '\n';
|
||||
if (l1 != CompactLatticeWeight::Zero()) {
|
||||
l2 = Divide(l5, l1, DIVIDE_LEFT);
|
||||
l3 = Divide(l6, l1, DIVIDE_LEFT);
|
||||
std::cout << "l2 = " << l2 << '\n';
|
||||
std::cout << "l3 = " << l3 << '\n';
|
||||
l4 = divisor(l2, l3); // make sure l2 is now one.
|
||||
std::cout << "l4 = " << l4 << '\n';
|
||||
KALDI_ASSERT(ApproxEqual(l4, CompactLatticeWeight::One()));
|
||||
} else {
|
||||
KALDI_ASSERT(l5 == CompactLatticeWeight::Zero()
|
||||
&& l6 == CompactLatticeWeight::Zero());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
int main() {
|
||||
fst::LatticeWeightTest();
|
||||
fst::CompactLatticeWeightTest();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,862 @@
|
||||
// fstext/lattice-weight.h
|
||||
// Copyright 2009-2012 Microsoft Corporation
|
||||
// Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#ifndef KALDI_FSTEXT_LATTICE_WEIGHT_H_
|
||||
#define KALDI_FSTEXT_LATTICE_WEIGHT_H_
|
||||
|
||||
#include "fst/fstlib.h"
|
||||
#include "base/kaldi-common.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
// Declare weight type for lattice... will import to namespace kaldi. has two
|
||||
// members, value1_ and value2_, of type BaseFloat (normally equals float). It
|
||||
// is basically the same as the tropical semiring on value1_+value2_, except it
|
||||
// keeps track of a and b separately. More precisely, it is equivalent to the
|
||||
// lexicographic semiring on (value1_+value2_), (value1_-value2_)
|
||||
|
||||
|
||||
template<class FloatType>
|
||||
class LatticeWeightTpl;
|
||||
|
||||
template <class FloatType>
|
||||
inline std::ostream &operator <<(std::ostream &strm, const LatticeWeightTpl<FloatType> &w);
|
||||
|
||||
template <class FloatType>
|
||||
inline std::istream &operator >>(std::istream &strm, LatticeWeightTpl<FloatType> &w);
|
||||
|
||||
|
||||
template<class FloatType>
|
||||
class LatticeWeightTpl {
|
||||
public:
|
||||
typedef FloatType T; // normally float.
|
||||
typedef LatticeWeightTpl ReverseWeight;
|
||||
|
||||
inline T Value1() const { return value1_; }
|
||||
|
||||
inline T Value2() const { return value2_; }
|
||||
|
||||
inline void SetValue1(T f) { value1_ = f; }
|
||||
|
||||
inline void SetValue2(T f) { value2_ = f; }
|
||||
|
||||
LatticeWeightTpl(): value1_{}, value2_{} { }
|
||||
|
||||
LatticeWeightTpl(T a, T b): value1_(a), value2_(b) {}
|
||||
|
||||
LatticeWeightTpl(const LatticeWeightTpl &other): value1_(other.value1_), value2_(other.value2_) { }
|
||||
|
||||
LatticeWeightTpl &operator=(const LatticeWeightTpl &w) {
|
||||
value1_ = w.value1_;
|
||||
value2_ = w.value2_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
LatticeWeightTpl<FloatType> Reverse() const {
|
||||
return *this;
|
||||
}
|
||||
|
||||
static const LatticeWeightTpl Zero() {
|
||||
return LatticeWeightTpl(std::numeric_limits<T>::infinity(),
|
||||
std::numeric_limits<T>::infinity());
|
||||
}
|
||||
|
||||
static const LatticeWeightTpl One() {
|
||||
return LatticeWeightTpl(0.0, 0.0);
|
||||
}
|
||||
|
||||
static const std::string &Type() {
|
||||
static const std::string type = (sizeof(T) == 4 ? "lattice4" : "lattice8") ;
|
||||
return type;
|
||||
}
|
||||
|
||||
static const LatticeWeightTpl NoWeight() {
|
||||
return LatticeWeightTpl(std::numeric_limits<FloatType>::quiet_NaN(),
|
||||
std::numeric_limits<FloatType>::quiet_NaN());
|
||||
}
|
||||
|
||||
bool Member() const {
|
||||
// value1_ == value1_ tests for NaN.
|
||||
// also test for no -inf, and either both or neither
|
||||
// must be +inf, and
|
||||
if (value1_ != value1_ || value2_ != value2_) return false; // NaN
|
||||
if (value1_ == -std::numeric_limits<T>::infinity() ||
|
||||
value2_ == -std::numeric_limits<T>::infinity()) return false; // -infty not allowed
|
||||
if (value1_ == std::numeric_limits<T>::infinity() ||
|
||||
value2_ == std::numeric_limits<T>::infinity()) {
|
||||
if (value1_ != std::numeric_limits<T>::infinity() ||
|
||||
value2_ != std::numeric_limits<T>::infinity()) return false; // both must be +infty;
|
||||
// this is necessary so that the semiring has only one zero.
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
LatticeWeightTpl Quantize(float delta = kDelta) const {
|
||||
if (value1_ + value2_ == -std::numeric_limits<T>::infinity()) {
|
||||
return LatticeWeightTpl(-std::numeric_limits<T>::infinity(), -std::numeric_limits<T>::infinity());
|
||||
} else if (value1_ + value2_ == std::numeric_limits<T>::infinity()) {
|
||||
return LatticeWeightTpl(std::numeric_limits<T>::infinity(), std::numeric_limits<T>::infinity());
|
||||
} else if (value1_ + value2_ != value1_ + value2_) { // NaN
|
||||
return LatticeWeightTpl(value1_ + value2_, value1_ + value2_);
|
||||
} else {
|
||||
return LatticeWeightTpl(floor(value1_/delta + 0.5F)*delta, floor(value2_/delta + 0.5F) * delta);
|
||||
}
|
||||
}
|
||||
static constexpr uint64 Properties() {
|
||||
return kLeftSemiring | kRightSemiring | kCommutative |
|
||||
kPath | kIdempotent;
|
||||
}
|
||||
|
||||
// This is used in OpenFst for binary I/O. This is OpenFst-style,
|
||||
// not Kaldi-style, I/O.
|
||||
std::istream &Read(std::istream &strm) {
|
||||
// Always read/write as float, even if T is double,
|
||||
// so we can use OpenFst-style read/write and still maintain
|
||||
// compatibility when compiling with different FloatTypes
|
||||
ReadType(strm, &value1_);
|
||||
ReadType(strm, &value2_);
|
||||
return strm;
|
||||
}
|
||||
|
||||
|
||||
// This is used in OpenFst for binary I/O. This is OpenFst-style,
|
||||
// not Kaldi-style, I/O.
|
||||
std::ostream &Write(std::ostream &strm) const {
|
||||
WriteType(strm, value1_);
|
||||
WriteType(strm, value2_);
|
||||
return strm;
|
||||
}
|
||||
|
||||
size_t Hash() const {
|
||||
size_t ans;
|
||||
union {
|
||||
T f;
|
||||
size_t s;
|
||||
} u;
|
||||
u.s = 0;
|
||||
u.f = value1_;
|
||||
ans = u.s;
|
||||
u.f = value2_;
|
||||
ans += u.s;
|
||||
return ans;
|
||||
}
|
||||
|
||||
protected:
|
||||
inline static void WriteFloatType(std::ostream &strm, const T &f) {
|
||||
if (f == std::numeric_limits<T>::infinity())
|
||||
strm << "Infinity";
|
||||
else if (f == -std::numeric_limits<T>::infinity())
|
||||
strm << "-Infinity";
|
||||
else if (f != f)
|
||||
strm << "BadNumber";
|
||||
else
|
||||
strm << f;
|
||||
}
|
||||
|
||||
// Internal helper function, used in ReadNoParen.
|
||||
inline static void ReadFloatType(std::istream &strm, T &f) {
|
||||
std::string s;
|
||||
strm >> s;
|
||||
if (s == "Infinity") {
|
||||
f = std::numeric_limits<T>::infinity();
|
||||
} else if (s == "-Infinity") {
|
||||
f = -std::numeric_limits<T>::infinity();
|
||||
} else if (s == "BadNumber") {
|
||||
f = std::numeric_limits<T>::quiet_NaN();
|
||||
} else {
|
||||
char *p;
|
||||
f = strtod(s.c_str(), &p);
|
||||
if (p < s.c_str() + s.size())
|
||||
strm.clear(std::ios::badbit);
|
||||
}
|
||||
}
|
||||
|
||||
// Reads LatticeWeight when there are no parentheses around pair terms...
|
||||
// currently the only form supported.
|
||||
inline std::istream &ReadNoParen(
|
||||
std::istream &strm, char separator) {
|
||||
int c;
|
||||
do {
|
||||
c = strm.get();
|
||||
} while (isspace(c));
|
||||
|
||||
std::string s1;
|
||||
while (c != separator) {
|
||||
if (c == EOF) {
|
||||
strm.clear(std::ios::badbit);
|
||||
return strm;
|
||||
}
|
||||
s1 += c;
|
||||
c = strm.get();
|
||||
}
|
||||
std::istringstream strm1(s1);
|
||||
ReadFloatType(strm1, value1_); // ReadFloatType is class member function
|
||||
// read second element
|
||||
ReadFloatType(strm, value2_);
|
||||
return strm;
|
||||
}
|
||||
|
||||
friend std::istream &operator>> <FloatType>(std::istream&, LatticeWeightTpl<FloatType>&);
|
||||
friend std::ostream &operator<< <FloatType>(std::ostream&, const LatticeWeightTpl<FloatType>&);
|
||||
|
||||
private:
|
||||
T value1_;
|
||||
T value2_;
|
||||
|
||||
|
||||
};
|
||||
|
||||
|
||||
/* ScaleTupleWeight is a function defined for LatticeWeightTpl and
|
||||
CompactLatticeWeightTpl that mutliplies the pair (value1_, value2_) by a 2x2
|
||||
matrix. Used, for example, in applying acoustic scaling.
|
||||
*/
|
||||
template<class FloatType, class ScaleFloatType>
|
||||
inline LatticeWeightTpl<FloatType> ScaleTupleWeight(
|
||||
const LatticeWeightTpl<FloatType> &w,
|
||||
const std::vector<std::vector<ScaleFloatType> > &scale) {
|
||||
// Without the next special case we'd get NaNs from infinity * 0
|
||||
if (w.Value1() == std::numeric_limits<FloatType>::infinity())
|
||||
return LatticeWeightTpl<FloatType>::Zero();
|
||||
return LatticeWeightTpl<FloatType>(scale[0][0] * w.Value1() + scale[0][1] * w.Value2(),
|
||||
scale[1][0] * w.Value1() + scale[1][1] * w.Value2());
|
||||
}
|
||||
|
||||
/* For testing purposes and in case it's ever useful, we define a similar
|
||||
function to apply to LexicographicWeight and the like, templated on
|
||||
TropicalWeight<float> etc.; we use PairWeight which is the base class of
|
||||
LexicographicWeight.
|
||||
*/
|
||||
template<class FloatType, class ScaleFloatType>
|
||||
inline PairWeight<TropicalWeightTpl<FloatType>,
|
||||
TropicalWeightTpl<FloatType> > ScaleTupleWeight(
|
||||
const PairWeight<TropicalWeightTpl<FloatType>,
|
||||
TropicalWeightTpl<FloatType> > &w,
|
||||
const std::vector<std::vector<ScaleFloatType> > &scale) {
|
||||
typedef TropicalWeightTpl<FloatType> BaseType;
|
||||
typedef PairWeight<BaseType, BaseType> PairType;
|
||||
const BaseType zero = BaseType::Zero();
|
||||
// Without the next special case we'd get NaNs from infinity * 0
|
||||
if (w.Value1() == zero || w.Value2() == zero)
|
||||
return PairType(zero, zero);
|
||||
FloatType f1 = w.Value1().Value(), f2 = w.Value2().Value();
|
||||
return PairType(BaseType(scale[0][0] * f1 + scale[0][1] * f2),
|
||||
BaseType(scale[1][0] * f1 + scale[1][1] * f2));
|
||||
}
|
||||
|
||||
|
||||
|
||||
template<class FloatType>
|
||||
inline bool operator==(const LatticeWeightTpl<FloatType> &wa,
|
||||
const LatticeWeightTpl<FloatType> &wb) {
|
||||
// Volatile qualifier thwarts over-aggressive compiler optimizations
|
||||
// that lead to problems esp. with NaturalLess().
|
||||
volatile FloatType va1 = wa.Value1(), va2 = wa.Value2(),
|
||||
vb1 = wb.Value1(), vb2 = wb.Value2();
|
||||
return (va1 == vb1 && va2 == vb2);
|
||||
}
|
||||
|
||||
template<class FloatType>
|
||||
inline bool operator!=(const LatticeWeightTpl<FloatType> &wa,
|
||||
const LatticeWeightTpl<FloatType> &wb) {
|
||||
// Volatile qualifier thwarts over-aggressive compiler optimizations
|
||||
// that lead to problems esp. with NaturalLess().
|
||||
volatile FloatType va1 = wa.Value1(), va2 = wa.Value2(),
|
||||
vb1 = wb.Value1(), vb2 = wb.Value2();
|
||||
return (va1 != vb1 || va2 != vb2);
|
||||
}
|
||||
|
||||
|
||||
// We define a Compare function LatticeWeightTpl even though it's
|
||||
// not required by the semiring standard-- it's just more efficient
|
||||
// to do it this way rather than using the NaturalLess template.
|
||||
|
||||
/// Compare returns -1 if w1 < w2, +1 if w1 > w2, and 0 if w1 == w2.
|
||||
|
||||
template<class FloatType>
|
||||
inline int Compare (const LatticeWeightTpl<FloatType> &w1,
|
||||
const LatticeWeightTpl<FloatType> &w2) {
|
||||
FloatType f1 = w1.Value1() + w1.Value2(),
|
||||
f2 = w2.Value1() + w2.Value2();
|
||||
if (f1 < f2) { return 1; } // having smaller cost means you're larger
|
||||
// in the semiring [higher probability]
|
||||
else if (f1 > f2) { return -1; }
|
||||
// mathematically we should be comparing (w1.value1_-w1.value2_ < w2.value1_-w2.value2_)
|
||||
// in the next line, but add w1.value1_+w1.value2_ = w2.value1_+w2.value2_ to both sides and
|
||||
// divide by two, and we get the simpler equivalent form w1.value1_ < w2.value1_.
|
||||
else if (w1.Value1() < w2.Value1()) { return 1; }
|
||||
else if (w1.Value1() > w2.Value1()) { return -1; }
|
||||
else { return 0; }
|
||||
}
|
||||
|
||||
|
||||
template<class FloatType>
|
||||
inline LatticeWeightTpl<FloatType> Plus(const LatticeWeightTpl<FloatType> &w1,
|
||||
const LatticeWeightTpl<FloatType> &w2) {
|
||||
return (Compare(w1, w2) >= 0 ? w1 : w2);
|
||||
}
|
||||
|
||||
|
||||
// For efficiency, override the NaturalLess template class.
|
||||
template<class FloatType>
|
||||
class NaturalLess<LatticeWeightTpl<FloatType> > {
|
||||
public:
|
||||
typedef LatticeWeightTpl<FloatType> Weight;
|
||||
|
||||
NaturalLess() {}
|
||||
|
||||
bool operator()(const Weight &w1, const Weight &w2) const {
|
||||
// NaturalLess is a negative order (opposite to normal ordering).
|
||||
// This operator () corresponds to "<" in the negative order, which
|
||||
// corresponds to the ">" in the normal order.
|
||||
return (Compare(w1, w2) == 1);
|
||||
}
|
||||
};
|
||||
template<>
|
||||
class NaturalLess<LatticeWeightTpl<float> > {
|
||||
public:
|
||||
typedef LatticeWeightTpl<float> Weight;
|
||||
|
||||
NaturalLess() {}
|
||||
|
||||
bool operator()(const Weight &w1, const Weight &w2) const {
|
||||
// NaturalLess is a negative order (opposite to normal ordering).
|
||||
// This operator () corresponds to "<" in the negative order, which
|
||||
// corresponds to the ">" in the normal order.
|
||||
return (Compare(w1, w2) == 1);
|
||||
}
|
||||
};
|
||||
template<>
|
||||
class NaturalLess<LatticeWeightTpl<double> > {
|
||||
public:
|
||||
typedef LatticeWeightTpl<double> Weight;
|
||||
|
||||
NaturalLess() {}
|
||||
|
||||
bool operator()(const Weight &w1, const Weight &w2) const {
|
||||
// NaturalLess is a negative order (opposite to normal ordering).
|
||||
// This operator () corresponds to "<" in the negative order, which
|
||||
// corresponds to the ">" in the normal order.
|
||||
return (Compare(w1, w2) == 1);
|
||||
}
|
||||
};
|
||||
|
||||
template<class FloatType>
|
||||
inline LatticeWeightTpl<FloatType> Times(const LatticeWeightTpl<FloatType> &w1,
|
||||
const LatticeWeightTpl<FloatType> &w2) {
|
||||
return LatticeWeightTpl<FloatType>(w1.Value1()+w2.Value1(), w1.Value2()+w2.Value2());
|
||||
}
|
||||
|
||||
// divide w1 by w2 (on left/right/any doesn't matter as
|
||||
// commutative).
|
||||
template<class FloatType>
|
||||
inline LatticeWeightTpl<FloatType> Divide(const LatticeWeightTpl<FloatType> &w1,
|
||||
const LatticeWeightTpl<FloatType> &w2,
|
||||
DivideType typ = DIVIDE_ANY) {
|
||||
typedef FloatType T;
|
||||
T a = w1.Value1() - w2.Value1(), b = w1.Value2() - w2.Value2();
|
||||
if (a != a || b != b || a == -std::numeric_limits<T>::infinity()
|
||||
|| b == -std::numeric_limits<T>::infinity()) {
|
||||
KALDI_WARN << "LatticeWeightTpl::Divide, NaN or invalid number produced. "
|
||||
<< "[dividing by zero?] Returning zero";
|
||||
return LatticeWeightTpl<T>::Zero();
|
||||
}
|
||||
if (a == std::numeric_limits<T>::infinity() ||
|
||||
b == std::numeric_limits<T>::infinity())
|
||||
return LatticeWeightTpl<T>::Zero(); // not a valid number if only one is infinite.
|
||||
return LatticeWeightTpl<T>(a, b);
|
||||
}
|
||||
|
||||
|
||||
template<class FloatType>
|
||||
inline bool ApproxEqual(const LatticeWeightTpl<FloatType> &w1,
|
||||
const LatticeWeightTpl<FloatType> &w2,
|
||||
float delta = kDelta) {
|
||||
if (w1.Value1() == w2.Value1() && w1.Value2() == w2.Value2()) return true; // handles Zero().
|
||||
return (fabs((w1.Value1() + w1.Value2()) - (w2.Value1() + w2.Value2())) <= delta);
|
||||
}
|
||||
|
||||
template <class FloatType>
|
||||
inline std::ostream &operator <<(std::ostream &strm, const LatticeWeightTpl<FloatType> &w) {
|
||||
LatticeWeightTpl<FloatType>::WriteFloatType(strm, w.Value1());
|
||||
CHECK(FLAGS_fst_weight_separator.size() == 1);
|
||||
strm << FLAGS_fst_weight_separator[0]; // comma by default;
|
||||
// may or may not be settable from Kaldi programs.
|
||||
LatticeWeightTpl<FloatType>::WriteFloatType(strm, w.Value2());
|
||||
return strm;
|
||||
}
|
||||
|
||||
template <class FloatType>
|
||||
inline std::istream &operator >>(std::istream &strm, LatticeWeightTpl<FloatType> &w1) {
|
||||
CHECK(FLAGS_fst_weight_separator.size() == 1);
|
||||
// separator defaults to ','
|
||||
return w1.ReadNoParen(strm, FLAGS_fst_weight_separator[0]);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// CompactLattice will be an acceptor (accepting the words/output-symbols),
|
||||
// with the weights and input-symbol-seqs on the arcs.
|
||||
// There must be a total order on W. We assume for the sake of efficiency
|
||||
// that there is a function
|
||||
// Compare(W w1, W w2) that returns -1 if w1 < w2, +1 if w1 > w2, and
|
||||
// zero if w1 == w2, and Plus for type W returns (Compare(w1,w2) >= 0 ? w1 : w2).
|
||||
|
||||
template<class WeightType, class IntType>
|
||||
class CompactLatticeWeightTpl {
|
||||
public:
|
||||
typedef WeightType W;
|
||||
|
||||
typedef CompactLatticeWeightTpl<WeightType, IntType> ReverseWeight;
|
||||
|
||||
// Plus is like LexicographicWeight on the pair (weight_, string_), but where we
|
||||
// use standard lexicographic order on string_ [this is not the same as
|
||||
// NaturalLess on the StringWeight equivalent, which does not define a
|
||||
// total order].
|
||||
// Times, Divide obvious... (support both left & right division..)
|
||||
// CommonDivisor would need to be coded separately.
|
||||
|
||||
CompactLatticeWeightTpl() { }
|
||||
|
||||
CompactLatticeWeightTpl(const WeightType &w, const std::vector<IntType> &s):
|
||||
weight_(w), string_(s) { }
|
||||
|
||||
CompactLatticeWeightTpl(const CompactLatticeWeightTpl &compactLatticeWeightTpl) = default;
|
||||
|
||||
CompactLatticeWeightTpl &operator=(const CompactLatticeWeightTpl &w) = default;
|
||||
|
||||
const W &Weight() const { return weight_; }
|
||||
|
||||
const std::vector<IntType> &String() const { return string_; }
|
||||
|
||||
void SetWeight(const W &w) { weight_ = w; }
|
||||
|
||||
void SetString(const std::vector<IntType> &s) { string_ = s; }
|
||||
|
||||
static const CompactLatticeWeightTpl<WeightType, IntType> Zero() {
|
||||
return CompactLatticeWeightTpl<WeightType, IntType>(
|
||||
WeightType::Zero(), std::vector<IntType>());
|
||||
}
|
||||
|
||||
static const CompactLatticeWeightTpl<WeightType, IntType> One() {
|
||||
return CompactLatticeWeightTpl<WeightType, IntType>(
|
||||
WeightType::One(), std::vector<IntType>());
|
||||
}
|
||||
|
||||
inline static std::string GetIntSizeString() {
|
||||
char buf[2];
|
||||
buf[0] = '0' + sizeof(IntType);
|
||||
buf[1] = '\0';
|
||||
return buf;
|
||||
}
|
||||
static const std::string &Type() {
|
||||
static const std::string type = "compact" + WeightType::Type()
|
||||
+ GetIntSizeString();
|
||||
return type;
|
||||
}
|
||||
|
||||
static const CompactLatticeWeightTpl<WeightType, IntType> NoWeight() {
|
||||
return CompactLatticeWeightTpl<WeightType, IntType>(
|
||||
WeightType::NoWeight(), std::vector<IntType>());
|
||||
}
|
||||
|
||||
|
||||
CompactLatticeWeightTpl<WeightType, IntType> Reverse() const {
|
||||
size_t s = string_.size();
|
||||
std::vector<IntType> v(s);
|
||||
for(size_t i = 0; i < s; i++)
|
||||
v[i] = string_[s-i-1];
|
||||
return CompactLatticeWeightTpl<WeightType, IntType>(weight_, v);
|
||||
}
|
||||
|
||||
bool Member() const {
|
||||
// a semiring has only one zero, this is the important property
|
||||
// we're trying to maintain here. So force string_ to be empty if
|
||||
// w_ == zero.
|
||||
if (!weight_.Member()) return false;
|
||||
if (weight_ == WeightType::Zero())
|
||||
return string_.empty();
|
||||
else
|
||||
return true;
|
||||
}
|
||||
|
||||
CompactLatticeWeightTpl Quantize(float delta = kDelta) const {
|
||||
return CompactLatticeWeightTpl(weight_.Quantize(delta), string_);
|
||||
}
|
||||
|
||||
static constexpr uint64 Properties() {
|
||||
return kLeftSemiring | kRightSemiring | kPath | kIdempotent;
|
||||
}
|
||||
|
||||
// This is used in OpenFst for binary I/O. This is OpenFst-style,
|
||||
// not Kaldi-style, I/O.
|
||||
std::istream &Read(std::istream &strm) {
|
||||
weight_.Read(strm);
|
||||
if (strm.fail()){ return strm; }
|
||||
int32 sz;
|
||||
ReadType(strm, &sz);
|
||||
if (strm.fail()){ return strm; }
|
||||
if (sz < 0) {
|
||||
KALDI_WARN << "Negative string size! Read failure";
|
||||
strm.clear(std::ios::badbit);
|
||||
return strm;
|
||||
}
|
||||
string_.resize(sz);
|
||||
for(int32 i = 0; i < sz; i++) {
|
||||
ReadType(strm, &(string_[i]));
|
||||
}
|
||||
return strm;
|
||||
}
|
||||
|
||||
// This is used in OpenFst for binary I/O. This is OpenFst-style,
|
||||
// not Kaldi-style, I/O.
|
||||
std::ostream &Write(std::ostream &strm) const {
|
||||
weight_.Write(strm);
|
||||
if (strm.fail()){ return strm; }
|
||||
int32 sz = static_cast<int32>(string_.size());
|
||||
WriteType(strm, sz);
|
||||
for(int32 i = 0; i < sz; i++)
|
||||
WriteType(strm, string_[i]);
|
||||
return strm;
|
||||
}
|
||||
size_t Hash() const {
|
||||
size_t ans = weight_.Hash();
|
||||
// any weird numbers here are largish primes
|
||||
size_t sz = string_.size(), mult = 6967;
|
||||
for(size_t i = 0; i < sz; i++) {
|
||||
ans += string_[i] * mult;
|
||||
mult *= 7499;
|
||||
}
|
||||
return ans;
|
||||
}
|
||||
private:
|
||||
W weight_;
|
||||
std::vector<IntType> string_;
|
||||
|
||||
};
|
||||
|
||||
template<class WeightType, class IntType>
|
||||
inline bool operator==(const CompactLatticeWeightTpl<WeightType, IntType> &w1,
|
||||
const CompactLatticeWeightTpl<WeightType, IntType> &w2) {
|
||||
return (w1.Weight() == w2.Weight() && w1.String() == w2.String());
|
||||
}
|
||||
|
||||
template<class WeightType, class IntType>
|
||||
inline bool operator!=(const CompactLatticeWeightTpl<WeightType, IntType> &w1,
|
||||
const CompactLatticeWeightTpl<WeightType, IntType> &w2) {
|
||||
return (w1.Weight() != w2.Weight() || w1.String() != w2.String());
|
||||
}
|
||||
|
||||
template<class WeightType, class IntType>
|
||||
inline bool ApproxEqual(const CompactLatticeWeightTpl<WeightType, IntType> &w1,
|
||||
const CompactLatticeWeightTpl<WeightType, IntType> &w2,
|
||||
float delta = kDelta) {
|
||||
return (ApproxEqual(w1.Weight(), w2.Weight(), delta) && w1.String() == w2.String());
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Compare is not part of the standard for weight types, but used internally for
|
||||
// efficiency. The comparison here first compares the weight; if this is the
|
||||
// same, it compares the string. The comparison on strings is: first compare
|
||||
// the length, if this is the same, use lexicographical order. We can't just
|
||||
// use the lexicographical order because this would destroy the distributive
|
||||
// property of multiplication over addition, taking into account that addition
|
||||
// uses Compare. The string element of "Compare" isn't super-important in
|
||||
// practical terms; it's only needed to ensure that Plus always give consistent
|
||||
// answers and is symmetric. It's essentially for tie-breaking, but we need to
|
||||
// make sure all the semiring axioms are satisfied otherwise OpenFst might
|
||||
// break.
|
||||
|
||||
template<class WeightType, class IntType>
|
||||
inline int Compare(const CompactLatticeWeightTpl<WeightType, IntType> &w1,
|
||||
const CompactLatticeWeightTpl<WeightType, IntType> &w2) {
|
||||
int c1 = Compare(w1.Weight(), w2.Weight());
|
||||
if (c1 != 0) return c1;
|
||||
int l1 = w1.String().size(), l2 = w2.String().size();
|
||||
// Use opposite order on the string lengths, so that if the costs are the same,
|
||||
// the shorter string wins.
|
||||
if (l1 > l2) return -1;
|
||||
else if (l1 < l2) return 1;
|
||||
for(int i = 0; i < l1; i++) {
|
||||
if (w1.String()[i] < w2.String()[i]) return -1;
|
||||
else if (w1.String()[i] > w2.String()[i]) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// For efficiency, override the NaturalLess template class.
|
||||
template<class FloatType, class IntType>
|
||||
class NaturalLess<CompactLatticeWeightTpl<LatticeWeightTpl<FloatType>, IntType> > {
|
||||
public:
|
||||
typedef CompactLatticeWeightTpl<LatticeWeightTpl<FloatType>, IntType> Weight;
|
||||
|
||||
NaturalLess() {}
|
||||
|
||||
bool operator()(const Weight &w1, const Weight &w2) const {
|
||||
// NaturalLess is a negative order (opposite to normal ordering).
|
||||
// This operator () corresponds to "<" in the negative order, which
|
||||
// corresponds to the ">" in the normal order.
|
||||
return (Compare(w1, w2) == 1);
|
||||
}
|
||||
};
|
||||
template<>
|
||||
class NaturalLess<CompactLatticeWeightTpl<LatticeWeightTpl<float>, int32> > {
|
||||
public:
|
||||
typedef CompactLatticeWeightTpl<LatticeWeightTpl<float>, int32> Weight;
|
||||
|
||||
NaturalLess() {}
|
||||
|
||||
bool operator()(const Weight &w1, const Weight &w2) const {
|
||||
// NaturalLess is a negative order (opposite to normal ordering).
|
||||
// This operator () corresponds to "<" in the negative order, which
|
||||
// corresponds to the ">" in the normal order.
|
||||
return (Compare(w1, w2) == 1);
|
||||
}
|
||||
};
|
||||
template<>
|
||||
class NaturalLess<CompactLatticeWeightTpl<LatticeWeightTpl<double>, int32> > {
|
||||
public:
|
||||
typedef CompactLatticeWeightTpl<LatticeWeightTpl<double>, int32> Weight;
|
||||
|
||||
NaturalLess() {}
|
||||
|
||||
bool operator()(const Weight &w1, const Weight &w2) const {
|
||||
// NaturalLess is a negative order (opposite to normal ordering).
|
||||
// This operator () corresponds to "<" in the negative order, which
|
||||
// corresponds to the ">" in the normal order.
|
||||
return (Compare(w1, w2) == 1);
|
||||
}
|
||||
};
|
||||
|
||||
// Make sure Compare is defined for TropicalWeight, so everything works
|
||||
// if we substitute LatticeWeight for TropicalWeight.
|
||||
inline int Compare(const TropicalWeight &w1,
|
||||
const TropicalWeight &w2) {
|
||||
float f1 = w1.Value(), f2 = w2.Value();
|
||||
if (f1 == f2) return 0;
|
||||
else if (f1 > f2) return -1;
|
||||
else return 1;
|
||||
}
|
||||
|
||||
|
||||
|
||||
template<class WeightType, class IntType>
|
||||
inline CompactLatticeWeightTpl<WeightType, IntType> Plus(
|
||||
const CompactLatticeWeightTpl<WeightType, IntType> &w1,
|
||||
const CompactLatticeWeightTpl<WeightType, IntType> &w2) {
|
||||
return (Compare(w1, w2) >= 0 ? w1 : w2);
|
||||
}
|
||||
|
||||
template<class WeightType, class IntType>
|
||||
inline CompactLatticeWeightTpl<WeightType, IntType> Times(
|
||||
const CompactLatticeWeightTpl<WeightType, IntType> &w1,
|
||||
const CompactLatticeWeightTpl<WeightType, IntType> &w2) {
|
||||
WeightType w = Times(w1.Weight(), w2.Weight());
|
||||
if (w == WeightType::Zero()) {
|
||||
return CompactLatticeWeightTpl<WeightType, IntType>::Zero();
|
||||
// special case to ensure zero is unique
|
||||
} else {
|
||||
std::vector<IntType> v;
|
||||
v.resize(w1.String().size() + w2.String().size());
|
||||
typename std::vector<IntType>::iterator iter = v.begin();
|
||||
iter = std::copy(w1.String().begin(), w1.String().end(), iter); // returns end of first range.
|
||||
std::copy(w2.String().begin(), w2.String().end(), iter);
|
||||
return CompactLatticeWeightTpl<WeightType, IntType>(w, v);
|
||||
}
|
||||
}
|
||||
|
||||
template<class WeightType, class IntType>
|
||||
inline CompactLatticeWeightTpl<WeightType, IntType> Divide(const CompactLatticeWeightTpl<WeightType, IntType> &w1,
|
||||
const CompactLatticeWeightTpl<WeightType, IntType> &w2,
|
||||
DivideType div = DIVIDE_ANY) {
|
||||
if (w1.Weight() == WeightType::Zero()) {
|
||||
if (w2.Weight() != WeightType::Zero()) {
|
||||
return CompactLatticeWeightTpl<WeightType, IntType>::Zero();
|
||||
} else {
|
||||
KALDI_ERR << "Division by zero [0/0]";
|
||||
}
|
||||
} else if (w2.Weight() == WeightType::Zero()) {
|
||||
KALDI_ERR << "Error: division by zero";
|
||||
}
|
||||
WeightType w = Divide(w1.Weight(), w2.Weight());
|
||||
|
||||
const std::vector<IntType> v1 = w1.String(), v2 = w2.String();
|
||||
if (v2.size() > v1.size()) {
|
||||
KALDI_ERR << "Cannot divide, length mismatch";
|
||||
}
|
||||
typename std::vector<IntType>::const_iterator v1b = v1.begin(),
|
||||
v1e = v1.end(), v2b = v2.begin(), v2e = v2.end();
|
||||
if (div == DIVIDE_LEFT) {
|
||||
if (!std::equal(v2b, v2e, v1b)) { // v2 must be identical to first part of v1.
|
||||
KALDI_ERR << "Cannot divide, data mismatch";
|
||||
}
|
||||
return CompactLatticeWeightTpl<WeightType, IntType>(
|
||||
w, std::vector<IntType>(v1b+(v2e-v2b), v1e)); // return last part of v1.
|
||||
} else if (div == DIVIDE_RIGHT) {
|
||||
if (!std::equal(v2b, v2e, v1e-(v2e-v2b))) { // v2 must be identical to last part of v1.
|
||||
KALDI_ERR << "Cannot divide, data mismatch";
|
||||
}
|
||||
return CompactLatticeWeightTpl<WeightType, IntType>(
|
||||
w, std::vector<IntType>(v1b, v1e-(v2e-v2b))); // return first part of v1.
|
||||
|
||||
} else {
|
||||
KALDI_ERR << "Cannot divide CompactLatticeWeightTpl with DIVIDE_ANY";
|
||||
}
|
||||
return CompactLatticeWeightTpl<WeightType,IntType>::Zero(); // keep compiler happy.
|
||||
}
|
||||
|
||||
template <class WeightType, class IntType>
|
||||
inline std::ostream &operator <<(std::ostream &strm, const CompactLatticeWeightTpl<WeightType, IntType> &w) {
|
||||
strm << w.Weight();
|
||||
CHECK(FLAGS_fst_weight_separator.size() == 1);
|
||||
strm << FLAGS_fst_weight_separator[0]; // comma by default.
|
||||
for(size_t i = 0; i < w.String().size(); i++) {
|
||||
strm << w.String()[i];
|
||||
if (i+1 < w.String().size())
|
||||
strm << kStringSeparator; // '_'; defined in string-weight.h in OpenFst code.
|
||||
}
|
||||
return strm;
|
||||
}
|
||||
|
||||
template <class WeightType, class IntType>
|
||||
inline std::istream &operator >>(std::istream &strm, CompactLatticeWeightTpl<WeightType, IntType> &w) {
|
||||
std::string s;
|
||||
strm >> s;
|
||||
if (strm.fail()) {
|
||||
return strm;
|
||||
}
|
||||
CHECK(FLAGS_fst_weight_separator.size() == 1);
|
||||
size_t pos = s.find_last_of(FLAGS_fst_weight_separator); // normally ","
|
||||
if (pos == std::string::npos) {
|
||||
strm.clear(std::ios::badbit);
|
||||
return strm;
|
||||
}
|
||||
// get parts of str before and after the separator (default: ',');
|
||||
std::string s1(s, 0, pos), s2(s, pos+1);
|
||||
std::istringstream strm1(s1);
|
||||
WeightType weight;
|
||||
strm1 >> weight;
|
||||
w.SetWeight(weight);
|
||||
if (strm1.fail() || !strm1.eof()) {
|
||||
strm.clear(std::ios::badbit);
|
||||
return strm;
|
||||
}
|
||||
// read string part.
|
||||
std::vector<IntType> string;
|
||||
const char *c = s2.c_str();
|
||||
while(*c != '\0') {
|
||||
if (*c == kStringSeparator) // '_'
|
||||
c++;
|
||||
char *c2;
|
||||
long int i = strtol(c, &c2, 10);
|
||||
if (c2 == c || static_cast<long int>(static_cast<IntType>(i)) != i) {
|
||||
strm.clear(std::ios::badbit);
|
||||
return strm;
|
||||
}
|
||||
c = c2;
|
||||
string.push_back(static_cast<IntType>(i));
|
||||
}
|
||||
w.SetString(string);
|
||||
return strm;
|
||||
}
|
||||
|
||||
template<class BaseWeightType, class IntType>
|
||||
class CompactLatticeWeightCommonDivisorTpl {
|
||||
public:
|
||||
typedef CompactLatticeWeightTpl<BaseWeightType, IntType> Weight;
|
||||
|
||||
Weight operator()(const Weight &w1, const Weight &w2) const {
|
||||
// First find longest common prefix of the strings.
|
||||
typename std::vector<IntType>::const_iterator s1b = w1.String().begin(),
|
||||
s1e = w1.String().end(), s2b = w2.String().begin(), s2e = w2.String().end();
|
||||
while (s1b < s1e && s2b < s2e && *s1b == *s2b) {
|
||||
s1b++;
|
||||
s2b++;
|
||||
}
|
||||
return Weight(Plus(w1.Weight(), w2.Weight()), std::vector<IntType>(w1.String().begin(), s1b));
|
||||
}
|
||||
};
|
||||
|
||||
/** Scales the pair (a, b) of floating-point weights inside a
|
||||
CompactLatticeWeight by premultiplying it (viewed as a vector)
|
||||
by a 2x2 matrix "scale".
|
||||
Assumes there is a ScaleTupleWeight function that applies to "Weight";
|
||||
this currently only works if Weight equals LatticeWeightTpl<FloatType>
|
||||
for some FloatType.
|
||||
*/
|
||||
template<class Weight, class IntType, class ScaleFloatType>
|
||||
inline CompactLatticeWeightTpl<Weight, IntType> ScaleTupleWeight(
|
||||
const CompactLatticeWeightTpl<Weight, IntType> &w,
|
||||
const std::vector<std::vector<ScaleFloatType> > &scale) {
|
||||
return CompactLatticeWeightTpl<Weight, IntType>(
|
||||
Weight(ScaleTupleWeight(w.Weight(), scale)), w.String());
|
||||
}
|
||||
|
||||
/** Define some ConvertLatticeWeight functions that are used in various lattice
|
||||
conversions... make them all templates, some with no arguments, since some
|
||||
must be templates.*/
|
||||
template<class Float1, class Float2>
|
||||
inline void ConvertLatticeWeight(
|
||||
const LatticeWeightTpl<Float1> &w_in,
|
||||
LatticeWeightTpl<Float2> *w_out) {
|
||||
w_out->SetValue1(w_in.Value1());
|
||||
w_out->SetValue2(w_in.Value2());
|
||||
}
|
||||
|
||||
template<class Float1, class Float2, class Int>
|
||||
inline void ConvertLatticeWeight(
|
||||
const CompactLatticeWeightTpl<LatticeWeightTpl<Float1>, Int> &w_in,
|
||||
CompactLatticeWeightTpl<LatticeWeightTpl<Float2>, Int> *w_out) {
|
||||
LatticeWeightTpl<Float2> weight2(w_in.Weight().Value1(),
|
||||
w_in.Weight().Value2());
|
||||
w_out->SetWeight(weight2);
|
||||
w_out->SetString(w_in.String());
|
||||
}
|
||||
|
||||
// to convert from Lattice to standard FST
|
||||
template<class Float1, class Float2>
|
||||
inline void ConvertLatticeWeight(
|
||||
const LatticeWeightTpl<Float1> &w_in,
|
||||
TropicalWeightTpl<Float2> *w_out) {
|
||||
TropicalWeightTpl<Float2> w1(w_in.Value1());
|
||||
TropicalWeightTpl<Float2> w2(w_in.Value2());
|
||||
*w_out = Times(w1, w2);
|
||||
}
|
||||
|
||||
template<class Float>
|
||||
inline double ConvertToCost(const LatticeWeightTpl<Float> &w) {
|
||||
return static_cast<double>(w.Value1()) + static_cast<double>(w.Value2());
|
||||
}
|
||||
|
||||
template<class Float, class Int>
|
||||
inline double ConvertToCost(const CompactLatticeWeightTpl<LatticeWeightTpl<Float>, Int> &w) {
|
||||
return static_cast<double>(w.Weight().Value1()) + static_cast<double>(w.Weight().Value2());
|
||||
}
|
||||
|
||||
template<class Float>
|
||||
inline double ConvertToCost(const TropicalWeightTpl<Float> &w) {
|
||||
return w.Value();
|
||||
}
|
||||
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // KALDI_FSTEXT_LATTICE_WEIGHT_H_
|
||||
@@ -0,0 +1,728 @@
|
||||
// fstext/pre-determinize-inl.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_PRE_DETERMINIZE_INL_H_
|
||||
#define KALDI_FSTEXT_PRE_DETERMINIZE_INL_H_
|
||||
|
||||
|
||||
/* Do not include this file directly. It is an implementation file included by PreDeterminize.h */
|
||||
|
||||
/*
|
||||
Predeterminization
|
||||
|
||||
This is a function that makes an FST compactly determinizable by inserting symbols on the input
|
||||
side as necessary for disambiguation. Note that we do not treat epsilon as a real symbol
|
||||
when measuring determinizability in this sense. The extra symbols are added to the vocabulary,
|
||||
on the input side; these are of the form (prefix)1, (prefix)2, and so on without limit, where
|
||||
(prefix) is some prefix the user provides, e.g. '#' (the function checks that this will not
|
||||
lead to conflicts with symbols already in the FST). The function tells us how many such
|
||||
symbols it created.
|
||||
|
||||
Note that there is a paper "Generalized optimization algorithm for speech recognition
|
||||
transducers" by Allauzen and Mohri, that deals with a similar issue, but this is a very
|
||||
different algorithm that only aims to ensure determinizability, but not *compact*
|
||||
determinizability.
|
||||
|
||||
Our algorithm is slightly heuristic, and probably not optimal, but does ensure that the
|
||||
output is compactly determinizable, possibly at the expense of inserting unnecessary
|
||||
symbols. We considered more sophisticated algorithms, but these were extremely
|
||||
complicated and would give the same output for the kinds of inputs that we envisage.
|
||||
|
||||
Suppose the input FST is T. We want to ensure that in det(T), if we consider the
|
||||
states of det(T) as weighted subsets of states of T, each state of T only appears once
|
||||
in any given subset. This ensures that det(T) is no larger than T in an appropriate
|
||||
sense. The way we do this is as follows. We identify all states in T that have
|
||||
multiple input transitions (counting "being an initial state" as an input transition).
|
||||
Let's call these "problematic" states. For a problematic state p we stipulate that it
|
||||
can never appear in any state of det(T) unless that state equals (p, \bar{1}) [i.e. p,
|
||||
unweighted]. In order to ensure this, we insert input symbols on the transitions to these
|
||||
problematic states (this may necessitate adding extra states).
|
||||
We also stipulate that the path through det(T) should always be sufficient to tell us
|
||||
the path through T (and we insert extra symbols sufficient to make this so). This is to
|
||||
simplify the algorithm, so that we don't have to consider the output symbols or weights
|
||||
when predeterminizing.
|
||||
|
||||
The algorithm is as follows.
|
||||
|
||||
(A) Definitions
|
||||
|
||||
(i) Define a *problematic state* as a state that either has multiple input transitions,
|
||||
or is an initial state and has at least one input transition.
|
||||
|
||||
(ii) For an arc a, define:
|
||||
i[a] = input symbol on a
|
||||
o[a] = output symbol on a
|
||||
n[a] = dest-state of a
|
||||
p[a] = origin-state of a
|
||||
|
||||
For a state q, define
|
||||
E[q] = set of transitions leaving q.
|
||||
For a set of states Q, define
|
||||
E[Q] = set of transitions leaving some q in Q
|
||||
|
||||
(iii) For a state s, define Closure(s) as the union of state s, and all states t
|
||||
that are reachable via sequences of arcs a such that i[a]=epsilon and n[a] is
|
||||
not problematic.
|
||||
|
||||
For a set of states S, define Closure(S) as the union of the closures of
|
||||
states s in S.
|
||||
|
||||
(B) Inputs and outputs.
|
||||
|
||||
(i) Inputs and preconditions. Input is an FST, which should have a symbol table compiled into
|
||||
it, and a prefix (e.g. #) for symbols to be added. We check that the input FST is trim,
|
||||
and that it does not have any symbols that appear on its arcs, that are equal to the prefix
|
||||
followed by digits.
|
||||
|
||||
(ii) Outputs: The algorithm modifies the FST that is given to it, and returns the number of
|
||||
the highest numbered "extra symbol" inserted. The extra symbols are numbered #1, #2 and
|
||||
so on without limit (as integers). They are inserted into the symbol table in a sequential
|
||||
way by calling AvailableKey()
|
||||
for each in turn (this is stipulated in case we need to keep other symbol tables in sync).
|
||||
|
||||
(C) Sub-algorithm: Closure(S). This requires the array p(s), defined below, which is true
|
||||
if s is problematic. This also requires, for efficiency, that the arcs be sorted on input
|
||||
label.
|
||||
Input: a set of states S. [plus, the fst and the array p].
|
||||
Output: a set of states T.
|
||||
Algorithm:
|
||||
set T <-- S, Q <-- S.
|
||||
while Q is nonempty:
|
||||
pop a state s from Q.
|
||||
for each transition a from state s with epsilon on the input label [we can
|
||||
find these efficiently using the sorting on arcs]:
|
||||
If p(n[a]) is false and n[a] is not in T:
|
||||
Insert n[a] into T.
|
||||
Add n[a] to Q.
|
||||
return T.
|
||||
|
||||
|
||||
(D) Main algorithm.
|
||||
|
||||
|
||||
(i) (a) Check preconditions (FST is trim)
|
||||
(b) Make sure there is just one final state (insert epsilon transitions as necessary).
|
||||
(c) Sort arcs on input label (so epsilon arcs are at the start of arc lists).
|
||||
|
||||
|
||||
(ii) Work out the set of problematic states by constructing a boolean array indexed by
|
||||
states, i.e.
|
||||
p(s)
|
||||
which is true if the state is problematic. We can do this by constructing an array
|
||||
t(s) to store the number of transitions into each state [adding one for the initial state],
|
||||
and then setting p(s) = true if t(s) > 1.
|
||||
|
||||
Also create a boolean array d(s), defined for states, and set d(s) = false.
|
||||
This array is purely for sanity-checking that we are processing each state exactly once.
|
||||
|
||||
(iii) Set up an array of integers m(a), indexed by arcs (how exactly we store these is
|
||||
implementation-dependent, but this will probably be a hash from (state, arc-index) to
|
||||
integers. m(a) will store the extra symbol, if any, to be added to that arc (or -1
|
||||
if no such symbol; we can also simply have the arc not present in the hash). The
|
||||
initial value of m(a) is -1 (if array), or undefined (if hash).
|
||||
|
||||
(iv) Initialize a set of sets-of-states S, and a queue of pairs Q, as follows.
|
||||
The pairs in Q are a pair of (set-of-states, integer), where the integer
|
||||
is the number of "special symbols" already used up for that state.
|
||||
|
||||
Note that we use a special indexing for the sets in both S and Q, rather than
|
||||
using std::set. We use a sorted vector of StateId's. And in S, we index them
|
||||
by the lowest-numbered state-id. Because each state is supposed to only ever
|
||||
be a member of one set, if there is an attempt to add another, different set
|
||||
with the same lowest-numbered state-id, we detect an error.
|
||||
|
||||
Let I be the single initial state (OpenFST only supports one).
|
||||
We set:
|
||||
S = { Closure(I) }
|
||||
Push (Closure(I), 0) onto Q.
|
||||
Then for each state s such that p(s) = true, and s is not an initial state:
|
||||
S <-- S u { Closure(s) }
|
||||
Push (Closure(s), 0) onto Q.
|
||||
|
||||
(v) While Q is nonempty:
|
||||
|
||||
(a) Pop pair (A, n) from Q (queue discipline is arbitrary).
|
||||
|
||||
(b) For each state s in A, check that d(s) is false, and set d(s) to true.
|
||||
This is for sanity checking only.
|
||||
|
||||
(c)
|
||||
Let S_\eps be the set of epsilon-transitions from members of A to problematic
|
||||
states (i.e. S_\eps = \{ a \in E[A]: i[a]=\epsilon, p(n[a]) = true \}).
|
||||
|
||||
Next, we will define, for each t \neq \epsilon, S_t as the set of
|
||||
transitions from some state s in S with t as the input label, i.e.:
|
||||
S_t = \{ a \in E[A]: i[a] = t \}
|
||||
We further define T_t and U_t as the subsets of S where the destination
|
||||
state is problematic and non-problematic respectively, i.e:
|
||||
T_t = \{ a \in E[A]: i[a] = t, p(n[a]) = true \}
|
||||
U_t = \{ a \in E[A]: i[a] = t, p(n[a]) = false \}
|
||||
|
||||
The easiest way to obtain these sets is probably to have a hash indexed by
|
||||
t that maps to a list of pairs (state, arc-offset) that stores S_t.
|
||||
From this we can work out the sizes of T_t and U_t on the fly.
|
||||
|
||||
(d)
|
||||
for each transition a in S_\eps:
|
||||
m(a) <-- n # Will put symbol n on this transition.
|
||||
n <-- n+1 # Note, same n as in pair (A, n)
|
||||
|
||||
(e)
|
||||
next,
|
||||
for each t\neq epsilon s.t. S_t is nonempty,
|
||||
|
||||
if |S_t| > 1 #if-statement is because if |S_t|=|T_t|=1, no need for prefix.
|
||||
k = 0
|
||||
for each transition a in T_t:
|
||||
set m(a) to k.
|
||||
set k = k+1
|
||||
|
||||
if |U_t| > 0
|
||||
Let V_t be the set of destination-states of arcs in U_t.
|
||||
if Closure(V_t) is not in S:
|
||||
insert Closure(V_t) into S, and add the pair (Closure(V_t), k) to Q.
|
||||
|
||||
(vi) Check that for each state in the FST, d(s) = true.
|
||||
|
||||
(vii) Let n = max_a m(a). This is the highest-numbered extra symbol (extra symbols
|
||||
start from zero, in this numbering which doesn't correspond to the symbol-table
|
||||
numbering). Here we add n+1 extra symbols to the symbol table and store
|
||||
the mappings from 0, 1, ... n to the symbol-id.
|
||||
|
||||
(viii) Set up a hash h from (state, int) to (state-id) such that
|
||||
t = h(s, k)
|
||||
will be the state-id of a newly-created state that has a transition to state s
|
||||
with input-label #k.
|
||||
|
||||
(ix) For each arc a such that m(a) != 0:
|
||||
If i[a] = epsilon (the input label is epsilon):
|
||||
Change i[a] to #m(a). [i.e. prefix then digit m(a)]
|
||||
Otherwise:
|
||||
If t = h(n[a], m(a)) is not defined [where n[a] is the dest-state]:
|
||||
create a new state t with a transition to n[a], with input-label #m(a) and
|
||||
no output-label or weight. Set h(n[a], m(a)) = t.
|
||||
Change n[a] to h(n[a], m(a)).
|
||||
|
||||
|
||||
*/
|
||||
namespace fst {
|
||||
|
||||
namespace pre_determinize_helpers {
|
||||
|
||||
// make it inline to avoid having to put it in a .cc file which most functions here
|
||||
// could not go in.
|
||||
inline bool HasBannedPrefixPlusDigits(SymbolTable *symTable, std::string prefix, std::string *bad_sym) {
|
||||
// returns true if the symbol table contains any string consisting of this
|
||||
// (possibly empty) prefix followed by a nonempty sequence of digits (0 to 9).
|
||||
// requires symTable to be non-NULL.
|
||||
// if bad_sym != NULL, puts the first bad symbol it finds in *bad_sym.
|
||||
assert(symTable != NULL);
|
||||
const char *prefix_ptr = prefix.c_str();
|
||||
size_t prefix_len = strlen(prefix_ptr); // allowed to be zero but not encouraged.
|
||||
for (SymbolTableIterator siter(*symTable); !siter.Done(); siter.Next()) {
|
||||
const std::string &sym = siter.Symbol();
|
||||
if (!strncmp(prefix_ptr, sym.c_str(), prefix_len)) { // has prefix.
|
||||
if (isdigit(sym[prefix_len])) { // we don't allow prefix followed by a digit, as a symbol.
|
||||
// Has at least one digit.
|
||||
size_t pos;
|
||||
for (pos = prefix_len;sym[pos] != '\0'; pos++)
|
||||
if (!isdigit(sym[pos])) break;
|
||||
if (sym[pos] == '\0') { // All remaining characters were digits.
|
||||
if (bad_sym != NULL) *bad_sym = sym;
|
||||
return true;
|
||||
}
|
||||
} // else OK because prefix was followed by '\0' or a non-digit.
|
||||
}
|
||||
}
|
||||
return false; // doesn't have banned symbol.
|
||||
}
|
||||
|
||||
template<class T> void CopySetToVector(const std::set<T> s, std::vector<T> *v) {
|
||||
// adds members of s to v, in sorted order from lowest to highest
|
||||
// (because the set was in sorted order).
|
||||
assert(v != NULL);
|
||||
v->resize(s.size());
|
||||
typename std::set<T>::const_iterator siter = s.begin();
|
||||
typename std::vector<T>::iterator viter = v->begin();
|
||||
for (; siter != s.end(); ++siter, ++viter) {
|
||||
assert(viter != v->end());
|
||||
*viter = *siter;
|
||||
}
|
||||
}
|
||||
|
||||
// Warning. This function calls 'new'.
|
||||
template<class T>
|
||||
std::vector<T>* InsertMember(const std::vector<T> m, std::vector<std::vector<T>*> *S) {
|
||||
assert(m.size() > 0);
|
||||
T idx = m[0];
|
||||
assert(idx>=(T)0 && idx < (T)S->size());
|
||||
if ( (*S)[idx] != NULL) {
|
||||
assert( *((*S)[idx]) == m );
|
||||
// The vectors should be the same. Otherwise this is a bug in the algorithm.
|
||||
// It could either be a programming error or a deeper conceptual bug.
|
||||
return NULL; // nothing was inserted.
|
||||
} else {
|
||||
std::vector<T> *ret = (*S)[idx] = new std::vector<T>(m); // New copy of m.
|
||||
return ret; // was inserted.
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// See definition of Closure(S) in item A(iii) in the comment above. it's the set of states
|
||||
// that are reachable from S via sequences of arcs a such that i[a]=epsilon and n[a] is
|
||||
// not problematic. We assume that the fst is sorted on input label (so epsilon arcs first)
|
||||
// The algorithm is described in section (C) above. We use the same variable for S and T.
|
||||
template<class Arc> void Closure(MutableFst<Arc> *fst, std::set<typename Arc::StateId> *S,
|
||||
const std::vector<bool> &pVec) {
|
||||
typedef typename Arc::StateId StateId;
|
||||
std::vector<StateId> Q;
|
||||
CopySetToVector(*S, &Q);
|
||||
while (Q.size() != 0) {
|
||||
StateId s = Q.back();
|
||||
Q.pop_back();
|
||||
for (ArcIterator<MutableFst<Arc> > aiter(*fst, s); ! aiter.Done(); aiter.Next()) {
|
||||
const Arc &arc = aiter.Value();
|
||||
if (arc.ilabel != 0) break; // Break from the loop: due to sorting there will be no
|
||||
// more transitions with epsilons as input labels.
|
||||
if (!pVec[arc.nextstate]) { // Next state is not problematic -> we can use this transition.
|
||||
std::pair< typename std::set<StateId>::iterator, bool > p = S->insert(arc.nextstate);
|
||||
if (p.second) { // True means: was inserted into S (wasn't already there).
|
||||
Q.push_back(arc.nextstate);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} // end function Closure.
|
||||
|
||||
} // end namespace pre_determinize_helpers.
|
||||
|
||||
|
||||
template<class Arc, class Int>
|
||||
void PreDeterminize(MutableFst<Arc> *fst,
|
||||
typename Arc::Label first_new_sym,
|
||||
std::vector<Int> *symsOut) {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef size_t ArcId; // Our own typedef, not standard OpenFst. Use size_t
|
||||
// for compatibility with argument of ArcIterator::Seek().
|
||||
typedef typename Arc::Weight Weight;
|
||||
assert(first_new_sym > 0);
|
||||
assert(fst != NULL);
|
||||
if (fst->Start() == kNoStateId) return; // for empty FST, nothing to do.
|
||||
assert(symsOut != NULL && symsOut->size() == 0); // we will output the symbols we add into this.
|
||||
|
||||
{ // (D)(i)(a): check is trim (i.e. connected, in OpenFST parlance).
|
||||
KALDI_VLOG(2) << "PreDeterminize: Checking FST properties";
|
||||
uint64 props = fst->Properties(kAccessible|kCoAccessible, true); // true-> computes properties if unknown at time when called.
|
||||
if (props != (kAccessible|kCoAccessible)) { // All states are not both accessible and co-accessible...
|
||||
KALDI_ERR << "PreDeterminize: FST is not trim";
|
||||
}
|
||||
}
|
||||
|
||||
{ // (D)(i)(b): make single final state.
|
||||
KALDI_VLOG(2) << "PreDeterminize: creating single final state";
|
||||
CreateSuperFinal(fst);
|
||||
}
|
||||
|
||||
{ // (D)(i)(c): sort arcs on input.
|
||||
KALDI_VLOG(2) << "PreDeterminize: sorting arcs on input";
|
||||
ILabelCompare<Arc> icomp;
|
||||
ArcSort(fst, icomp);
|
||||
}
|
||||
|
||||
StateId n_states = 0, max_state = 0; // Compute n_states, max_state = highest-numbered state.
|
||||
{ // compute nStates, maxStates.
|
||||
for (StateIterator<MutableFst<Arc> > iter(*fst); ! iter.Done(); iter.Next()) {
|
||||
StateId state = iter.Value();
|
||||
assert(state>=0);
|
||||
n_states++;
|
||||
if (state > max_state) max_state = state;
|
||||
}
|
||||
KALDI_VLOG(2) << "PreDeterminize: n_states = "<<(n_states)<<", max_state ="<<(max_state);
|
||||
}
|
||||
|
||||
std::vector<bool> p_vec(max_state+1, false); // compute this next.
|
||||
{ // D(ii): computing the array p. ["problematic states, i.e. states with >1 input transition,
|
||||
// counting being the initial state as an input transition"].
|
||||
std::vector<bool> seen_vec(max_state+1, false); // rather than counting incoming transitions we just have a bool that says we saw at least one.
|
||||
|
||||
seen_vec[fst->Start()] = true;
|
||||
for (StateIterator<MutableFst<Arc> > siter(*fst); ! siter.Done(); siter.Next()) {
|
||||
for (ArcIterator<MutableFst<Arc> > aiter(*fst, siter.Value()); ! aiter.Done(); aiter.Next()) {
|
||||
const Arc &arc = aiter.Value();
|
||||
assert(arc.nextstate>=0&&arc.nextstate<max_state+1);
|
||||
if (seen_vec[arc.nextstate])
|
||||
p_vec[arc.nextstate] = true; // now have >1 transition in, so problematic.
|
||||
else
|
||||
seen_vec[arc.nextstate] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
// D(iii): set up m(a)
|
||||
std::map<std::pair<StateId, ArcId>, size_t> m_map;
|
||||
// This is the array m, indexed by arcs. It maps to the index of the symbol we add.
|
||||
|
||||
|
||||
// WARNING: we should be sure to clean up this memory before exiting. Do not return
|
||||
// or throw an exception from this function, later than this point, without cleaning up!
|
||||
// Note that the vectors are shared between Q and S (they "belong to" S.
|
||||
std::vector<std::vector<StateId>* > S(max_state+1, (std::vector<StateId>*)(void*)0);
|
||||
std::vector<std::pair<std::vector<StateId>*, size_t> > Q;
|
||||
|
||||
// D(iv): initialize S and Q.
|
||||
{
|
||||
std::vector<StateId> all_seed_states; // all "problematic" states, plus initial state (if not problematic).
|
||||
if (!p_vec[fst->Start()])
|
||||
all_seed_states.push_back(fst->Start());
|
||||
for (StateId s = 0;s<=max_state; s++)
|
||||
if (p_vec[s]) all_seed_states.push_back(s);
|
||||
|
||||
for (size_t idx = 0;idx < all_seed_states.size(); idx++) {
|
||||
StateId s = all_seed_states[idx];
|
||||
std::set<StateId> closure_s;
|
||||
closure_s.insert(s); // insert "seed" state.
|
||||
pre_determinize_helpers::Closure(fst, &closure_s, p_vec); // follow epsilons to non-problematic states.
|
||||
// Closure in this case whis will usually not add anything, for typical topologies in speech
|
||||
std::vector<StateId> closure_s_vec;
|
||||
pre_determinize_helpers::CopySetToVector(closure_s, &closure_s_vec);
|
||||
KALDI_ASSERT(closure_s_vec.size() != 0);
|
||||
std::vector<StateId> *ptr = pre_determinize_helpers::InsertMember(closure_s_vec, &S);
|
||||
KALDI_ASSERT(ptr != NULL); // Or conceptual bug or programming error.
|
||||
Q.push_back(std::pair<std::vector<StateId>*, size_t>(ptr, 0));
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<bool> d_vec(max_state+1, false); // "done vector". Purely for debugging.
|
||||
|
||||
|
||||
size_t num_extra_det_states = 0;
|
||||
|
||||
// (D)(v)
|
||||
while (Q.size() != 0) {
|
||||
|
||||
// (D)(v)(a)
|
||||
std::pair<std::vector<StateId>*, size_t> cur_pair(Q.back());
|
||||
Q.pop_back();
|
||||
const std::vector<StateId> &A(*cur_pair.first);
|
||||
size_t n =cur_pair.second; // next special symbol to add.
|
||||
|
||||
// (D)(v)(b)
|
||||
for (size_t idx = 0;idx < A.size(); idx++) {
|
||||
assert(d_vec[A[idx]] == false && "This state has been seen before. Algorithm error.");
|
||||
d_vec[A[idx]] = true;
|
||||
}
|
||||
|
||||
// From here is (D)(v)(c). We work out S_\eps and S_t (for t\neq eps)
|
||||
// simultaneously at first.
|
||||
std::map<Label, std::set<std::pair<std::pair<StateId, ArcId>, StateId> > > arc_hash;
|
||||
// arc_hash is a hash with info of all arcs from states in the set A to
|
||||
// non-problematic states.
|
||||
// It is a map from ilabel to pair(pair(start-state, arc-offset), end-state).
|
||||
// Here, arc-offset reflects the order in which we accessed the arc using the
|
||||
// ArcIterator (zero for the first arc).
|
||||
|
||||
|
||||
{ // This block sets up arc_hash
|
||||
for (size_t idx = 0;idx < A.size(); idx++) {
|
||||
StateId s = A[idx];
|
||||
assert(s>=0 && s<=max_state);
|
||||
ArcId arc_id = 0;
|
||||
for (ArcIterator<MutableFst<Arc> > aiter(*fst, s); ! aiter.Done(); aiter.Next(), ++arc_id) {
|
||||
const Arc &arc = aiter.Value();
|
||||
|
||||
std::pair<std::pair<StateId, ArcId>, StateId>
|
||||
this_pair(std::pair<StateId, ArcId>(s, arc_id), arc.nextstate);
|
||||
bool inserted = (arc_hash[arc.ilabel].insert(this_pair)).second;
|
||||
assert(inserted); // Otherwise we had a duplicate.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// (D)(v)(d)
|
||||
if (arc_hash.count(0) == 1) { // We have epsilon transitions out.
|
||||
std::set<std::pair<std::pair<StateId, ArcId>, StateId> > &eps_set = arc_hash[0];
|
||||
typedef typename std::set<std::pair<std::pair<StateId, ArcId>, StateId> >::iterator set_iter_t;
|
||||
for (set_iter_t siter = eps_set.begin(); siter != eps_set.end(); ++siter) {
|
||||
const std::pair<std::pair<StateId, ArcId>, StateId> &this_pr = *siter;
|
||||
if (p_vec[this_pr.second]) { // Eps-transition to problematic state.
|
||||
assert(m_map.count(this_pr.first) == 0);
|
||||
m_map[this_pr.first] = n;
|
||||
n++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// (D)(v)(e)
|
||||
{
|
||||
typedef typename std::map<Label, std::set<std::pair<std::pair<StateId, ArcId>, StateId> > >::iterator map_iter_t;
|
||||
typedef typename std::set<std::pair<std::pair<StateId, ArcId>, StateId> >::iterator set_iter_t2;
|
||||
for (map_iter_t miter = arc_hash.begin(); miter != arc_hash.end(); ++miter) {
|
||||
Label t = miter->first;
|
||||
std::set<std::pair<std::pair<StateId, ArcId>, StateId> > &S_t = miter->second;
|
||||
if (t != 0) { // For t != epsilon,
|
||||
std::set<StateId> V_t; // set of destination non-problem states. Will create this set now.
|
||||
|
||||
// exists_noproblem is true iff |U_t| > 0.
|
||||
size_t k = 0;
|
||||
|
||||
// First loop "for each transition a in T_t" (i.e. transitions to problematic states)
|
||||
// The if-statement if (|S_t|>1) is pushed inside the loop, as the loop also computes
|
||||
// the set V_t.
|
||||
for (set_iter_t2 siter = S_t.begin(); siter != S_t.end(); ++siter) {
|
||||
const std::pair<std::pair<StateId, ArcId>, StateId> &this_pr = *siter;
|
||||
if (p_vec[this_pr.second]) { // only consider problematic states (just set T_t)
|
||||
if (S_t.size() > 1) { // This is where we pushed the if-statement in.
|
||||
assert(m_map.count(this_pr.first) == 0);
|
||||
m_map[this_pr.first] = k;
|
||||
k++;
|
||||
num_extra_det_states++;
|
||||
}
|
||||
} else { // Create the set V_t.
|
||||
V_t.insert(this_pr.second);
|
||||
}
|
||||
}
|
||||
if (V_t.size() != 0) {
|
||||
pre_determinize_helpers::Closure(fst, &V_t, p_vec); // follow epsilons to non-problematic states.
|
||||
std::vector<StateId> closure_V_t_vec;
|
||||
pre_determinize_helpers::CopySetToVector(V_t, &closure_V_t_vec);
|
||||
std::vector<StateId> *ptr = pre_determinize_helpers::InsertMember(closure_V_t_vec, &S);
|
||||
if (ptr != NULL) { // was inserted.
|
||||
Q.push_back(std::pair<std::vector<StateId>*, size_t>(ptr, k));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} // end while (Q.size() != 0)
|
||||
|
||||
|
||||
{ // (D)(vi): Check that for each state in the FST, d(s) = true.
|
||||
for (StateIterator<MutableFst<Arc> > siter(*fst); ! siter.Done(); siter.Next()) {
|
||||
StateId val = siter.Value();
|
||||
assert(d_vec[val] == true);
|
||||
}
|
||||
}
|
||||
|
||||
{ // (D)(vii): compute symbol-table ID's.
|
||||
// sets up symsOut array.
|
||||
int64 n = -1;
|
||||
for (typename std::map<std::pair<StateId, ArcId>, size_t>::iterator m_iter = m_map.begin();
|
||||
m_iter != m_map.end();
|
||||
++m_iter) {
|
||||
n = std::max(n, (int64) m_iter->second); // m_iter->second is of type size_t.
|
||||
}
|
||||
// At this point n is the highest symbol-id (type size_t) of symbols we must add.
|
||||
n++; // This is now the number of symbols we must add.
|
||||
for (size_t i = 0;static_cast<int64>(i)<n;i++) symsOut->push_back(first_new_sym + i);
|
||||
}
|
||||
|
||||
// (D)(viii): set up hash.
|
||||
std::map<std::pair<StateId, size_t>, StateId> h_map;
|
||||
|
||||
{ // D(ix): add extra symbols! This is where the work gets done.
|
||||
|
||||
// Core part of this is below, search for (*)
|
||||
size_t n_states_added = 0;
|
||||
|
||||
for (typename std::map<std::pair<StateId, ArcId>, size_t>::iterator m_iter = m_map.begin();
|
||||
m_iter != m_map.end();
|
||||
++m_iter) {
|
||||
StateId state = m_iter->first.first;
|
||||
ArcId arcpos = m_iter->first.second;
|
||||
size_t m_a = m_iter->second;
|
||||
|
||||
MutableArcIterator<MutableFst<Arc> > aiter(fst, state);
|
||||
aiter.Seek(arcpos);
|
||||
Arc arc = aiter.Value();
|
||||
|
||||
// (*) core part here.
|
||||
if (arc.ilabel == 0)
|
||||
arc.ilabel = (*symsOut)[m_a];
|
||||
else {
|
||||
std::pair<StateId, size_t> pr(arc.nextstate, m_a);
|
||||
if (!h_map.count(pr)) {
|
||||
n_states_added++;
|
||||
StateId newstate = fst->AddState();
|
||||
assert(newstate>=0);
|
||||
Arc new_arc( (*symsOut)[m_a], (Label)0, Weight::One(), arc.nextstate);
|
||||
fst->AddArc(newstate, new_arc);
|
||||
h_map[pr] = newstate;
|
||||
}
|
||||
arc.nextstate = h_map[pr];
|
||||
}
|
||||
aiter.SetValue(arc);
|
||||
}
|
||||
|
||||
KALDI_VLOG(2) << "Added " <<(n_states_added)<<" new states and added/changed "<<(m_map.size())<<" arcs";
|
||||
|
||||
}
|
||||
// Now free up memory.
|
||||
for (size_t i = 0;i < S.size();i++)
|
||||
delete S[i];
|
||||
} // end function PreDeterminize
|
||||
|
||||
|
||||
template<class Label> void CreateNewSymbols(SymbolTable *input_sym_table, int nSym,
|
||||
std::string prefix, std::vector<Label> *symsOut) {
|
||||
// Creates nSym new symbols named (prefix)0, (prefix)1 and so on.
|
||||
// Crashes if it cannot create them because one or more of them were in the symbol
|
||||
// table already.
|
||||
assert(symsOut && symsOut->size() == 0);
|
||||
for (int i = 0;i < nSym;i++) {
|
||||
std::stringstream ss; ss << prefix << i;
|
||||
std::string str = ss.str();
|
||||
if (input_sym_table->Find(str) != -1) { // should not be present.
|
||||
}
|
||||
assert(symsOut);
|
||||
symsOut->push_back( (Label) input_sym_table->AddSymbol(str));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// see pre-determinize.h for documentation.
|
||||
template<class Arc> void AddSelfLoops(MutableFst<Arc> *fst, std::vector<typename Arc::Label> &isyms,
|
||||
std::vector<typename Arc::Label> &osyms) {
|
||||
assert(fst != NULL);
|
||||
assert(isyms.size() == osyms.size());
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
size_t n = isyms.size();
|
||||
if (n == 0) return; // Nothing to do.
|
||||
|
||||
// {
|
||||
// the following declarations and statements are for quick detection of these
|
||||
// symbols, which is purely for debugging/checking purposes.
|
||||
Label isyms_min = *std::min_element(isyms.begin(), isyms.end()),
|
||||
isyms_max = *std::max_element(isyms.begin(), isyms.end()),
|
||||
osyms_min = *std::min_element(osyms.begin(), osyms.end()),
|
||||
osyms_max = *std::max_element(osyms.begin(), osyms.end());
|
||||
std::set<Label> isyms_set, osyms_set;
|
||||
for (size_t i = 0; i < isyms.size(); i++) {
|
||||
assert(isyms[i] > 0 && osyms[i] > 0); // should not have epsilon or invalid symbols.
|
||||
isyms_set.insert(isyms[i]);
|
||||
osyms_set.insert(osyms[i]);
|
||||
}
|
||||
assert(isyms_set.size() == n && osyms_set.size() == n);
|
||||
// } end block.
|
||||
|
||||
for (StateIterator<MutableFst<Arc> > siter(*fst); ! siter.Done(); siter.Next()) {
|
||||
StateId state = siter.Value();
|
||||
bool this_state_needs_self_loops = (fst->Final(state) != Weight::Zero());
|
||||
for (ArcIterator<MutableFst<Arc> > aiter(*fst, state); ! aiter.Done(); aiter.Next()) {
|
||||
const Arc &arc = aiter.Value();
|
||||
// If one of the following asserts fails, it means that the input FST already had the symbols
|
||||
// we are inserting. This is contrary to the preconditions of this algorithm.
|
||||
assert(!(arc.ilabel>=isyms_min && arc.ilabel<=isyms_max && isyms_set.count(arc.ilabel) != 0));
|
||||
assert(!(arc.olabel>=osyms_min && arc.olabel<=osyms_max && osyms_set.count(arc.olabel) != 0));
|
||||
if (arc.olabel != 0) // Has non-epsilon output label -> need self loops.
|
||||
this_state_needs_self_loops = true;
|
||||
}
|
||||
if (this_state_needs_self_loops) {
|
||||
for (size_t i = 0;i < n;i++) {
|
||||
Arc arc;
|
||||
arc.ilabel = isyms[i];
|
||||
arc.olabel = osyms[i];
|
||||
arc.weight = Weight::One();
|
||||
arc.nextstate = state;
|
||||
fst->AddArc(state, arc);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
int64 DeleteISymbols(MutableFst<Arc> *fst, std::vector<typename Arc::Label> isyms) {
|
||||
|
||||
// We could do this using the Mapper concept, but this is much easier to understand.
|
||||
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
|
||||
int64 num_deleted = 0;
|
||||
|
||||
if (isyms.size() == 0) return 0;
|
||||
Label isyms_min = *std::min_element(isyms.begin(), isyms.end()),
|
||||
isyms_max = *std::max_element(isyms.begin(), isyms.end());
|
||||
bool isyms_consecutive = (isyms_max+1-isyms_min == static_cast<Label>(isyms.size()));
|
||||
std::set<Label> isyms_set;
|
||||
if (!isyms_consecutive)
|
||||
for (size_t i = 0;i < isyms.size();i++)
|
||||
isyms_set.insert(isyms[i]);
|
||||
|
||||
for (StateIterator<MutableFst<Arc> > siter(*fst); ! siter.Done(); siter.Next()) {
|
||||
StateId state = siter.Value();
|
||||
for (MutableArcIterator<MutableFst<Arc> > aiter(fst, state); ! aiter.Done(); aiter.Next()) {
|
||||
const Arc &arc = aiter.Value();
|
||||
if (arc.ilabel >= isyms_min && arc.ilabel <= isyms_max) {
|
||||
if (isyms_consecutive || isyms_set.count(arc.ilabel) != 0) {
|
||||
num_deleted++;
|
||||
Arc mod_arc (arc);
|
||||
mod_arc.ilabel = 0; // change label to epsilon.
|
||||
aiter.SetValue(mod_arc);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return num_deleted;
|
||||
}
|
||||
|
||||
template<class Arc>
|
||||
typename Arc::StateId CreateSuperFinal(MutableFst<Arc> *fst) {
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
assert(fst != NULL);
|
||||
StateId num_states = fst->NumStates();
|
||||
std::vector<StateId> final_states;
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
if (fst->Final(s) != Weight::Zero()) {
|
||||
final_states.push_back(s);
|
||||
}
|
||||
}
|
||||
if (final_states.size() == 1) {
|
||||
if (fst->Final(final_states[0]) == Weight::One()) {
|
||||
ArcIterator<MutableFst<Arc> > iter(*fst, final_states[0]);
|
||||
if (iter.Done()) {
|
||||
// We already have a final state w/ no transitions out and unit weight.
|
||||
// So we're done.
|
||||
return final_states[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
StateId final_state = fst->AddState();
|
||||
fst->SetFinal(final_state, Weight::One());
|
||||
for (size_t idx = 0; idx < final_states.size(); idx++) {
|
||||
StateId s = final_states[idx];
|
||||
Weight weight = fst->Final(s);
|
||||
fst->SetFinal(s, Weight::Zero());
|
||||
Arc arc;
|
||||
arc.ilabel = 0;
|
||||
arc.olabel = 0;
|
||||
arc.nextstate = final_state;
|
||||
arc.weight = weight;
|
||||
fst->AddArc(s, arc);
|
||||
}
|
||||
return final_state;
|
||||
}
|
||||
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // KALDI_FSTEXT_PRE_DETERMINIZE_INL_H_
|
||||
@@ -0,0 +1,220 @@
|
||||
// fstext/pre-determinize-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "base/kaldi-math.h"
|
||||
#include "fstext/pre-determinize.h"
|
||||
#include "fstext/fst-test-utils.h"
|
||||
#include "fstext/fstext-utils.h"
|
||||
|
||||
// Just check that it compiles, for now.
|
||||
|
||||
namespace fst
|
||||
{
|
||||
using std::vector;
|
||||
using std::cout;
|
||||
|
||||
// Don't instantiate with log semiring, as RandEquivalent may fail.
|
||||
template<class Arc> void TestPreDeterminize() {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> *fst = new VectorFst<Arc>();
|
||||
int n_syms = 2 + kaldi::Rand() % 5, n_states = 3 + kaldi::Rand() % 10, n_arcs = 5 + kaldi::Rand() % 30, n_final = 1 + kaldi::Rand()%3; // Up to 2 unique symbols.
|
||||
cout << "Testing pre-determinize with "<<n_syms<<" symbols, "<<n_states<<" states and "<<n_arcs<<" arcs and "<<n_final<<" final states.\n";
|
||||
SymbolTable *sptr = NULL;
|
||||
|
||||
vector<Label> all_syms; // including epsilon.
|
||||
// Put symbols in the symbol table from 1..n_syms-1.
|
||||
for (size_t i = 0;i < (size_t)n_syms;i++)
|
||||
all_syms.push_back(i);
|
||||
|
||||
// Create states.
|
||||
vector<StateId> all_states;
|
||||
for (size_t i = 0;i < (size_t)n_states;i++) {
|
||||
StateId this_state = fst->AddState();
|
||||
if (i == 0) fst->SetStart(i);
|
||||
all_states.push_back(this_state);
|
||||
}
|
||||
// Set final states.
|
||||
for (size_t j = 0;j < (size_t)n_final;j++) {
|
||||
StateId id = all_states[kaldi::Rand() % n_states];
|
||||
Weight weight = (Weight)(0.33*(kaldi::Rand() % 5) );
|
||||
printf("calling SetFinal with %d and %f\n", id, weight.Value());
|
||||
fst->SetFinal(id, weight);
|
||||
}
|
||||
// Create arcs.
|
||||
for (size_t i = 0;i < (size_t)n_arcs;i++) {
|
||||
Arc a;
|
||||
a.nextstate = all_states[kaldi::Rand() % n_states];
|
||||
a.ilabel = all_syms[kaldi::Rand() % n_syms];
|
||||
a.olabel = all_syms[kaldi::Rand() % n_syms]; // same input+output vocab.
|
||||
a.weight = (Weight) (0.33*(kaldi::Rand() % 2));
|
||||
StateId start_state = all_states[kaldi::Rand() % n_states];
|
||||
fst->AddArc(start_state, a);
|
||||
}
|
||||
|
||||
std::cout <<" printing before trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
// Trim resulting FST.
|
||||
Connect(fst);
|
||||
|
||||
std::cout <<" printing after trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
VectorFst<Arc> *fst_copy_orig = new VectorFst<Arc>(*fst);
|
||||
|
||||
vector<Label> extra_syms;
|
||||
if (fst->Start() != kNoStateId) { // "Connect" did not make it empty....
|
||||
typename Arc::Label highest_sym = HighestNumberedInputSymbol(*fst);
|
||||
PreDeterminize(fst, highest_sym+1, &extra_syms);
|
||||
}
|
||||
|
||||
std::cout <<" printing after predeterminization\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
{ // Remove epsilon. All default args.
|
||||
bool connect = true;
|
||||
Weight weight_threshold = Weight::Zero();
|
||||
int64 nstate = -1; // Relates to pruning.
|
||||
double delta = kDelta; // I think a small weight value. Relates to some kind of pruning,
|
||||
// I guess. But with no epsilon cycles, probably doensn't matter.
|
||||
RmEpsilon(fst, connect, weight_threshold, nstate, delta);
|
||||
}
|
||||
|
||||
std::cout <<" printing after epsilon removal\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
VectorFst<Arc> ofst;
|
||||
DeterminizeOptions<Arc> opts; // Default options.
|
||||
Determinize(*fst, &ofst, opts);
|
||||
std::cout <<" printing after determinization\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(ofst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
int64 num_removed = DeleteISymbols(&ofst, extra_syms);
|
||||
std::cout <<" printing after removing "<<num_removed<<" instances of extra symbols\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(ofst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
std::cout <<" Checking equivalent to original FST.\n";
|
||||
// giving Rand() as a seed stops the random number generator from always being reset to
|
||||
// the same point each time, while maintaining determinism of the test.
|
||||
assert(RandEquivalent(ofst, *fst_copy_orig, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
delete fst;
|
||||
delete fst_copy_orig;
|
||||
}
|
||||
|
||||
template<class Arc> void TestAddSelfLoops() {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> *fst = new VectorFst<Arc>();
|
||||
SymbolTable *ilabels = new SymbolTable("my-symbol-table");
|
||||
SymbolTable *olabels = new SymbolTable("my-symbol-table-2");
|
||||
Label i0 = ilabels->AddSymbol("<eps>");
|
||||
Label i1 = ilabels->AddSymbol("1");
|
||||
Label i2 = ilabels->AddSymbol("2");
|
||||
|
||||
Label o0 = olabels->AddSymbol("<eps>");
|
||||
Label o1 = olabels->AddSymbol("1");
|
||||
|
||||
assert(i0 == 0 && o0 == 0);
|
||||
StateId s0 = fst->AddState(), s1 = fst->AddState(), s2 = fst->AddState();
|
||||
fst->SetStart(s0);
|
||||
assert(s0 == 0);
|
||||
|
||||
fst->SetFinal(s2, (Weight)2); // state 2 is final.
|
||||
{
|
||||
Arc arc;
|
||||
arc.ilabel = i1;
|
||||
arc.olabel = o0;
|
||||
arc.nextstate = 1;
|
||||
arc.weight = (Weight)1;
|
||||
fst->AddArc(s0, arc); // arc from 0 to 1 with epsilon out.
|
||||
}
|
||||
{
|
||||
Arc arc;
|
||||
arc.ilabel = i2;
|
||||
arc.olabel = o1;
|
||||
arc.nextstate = 2;
|
||||
arc.weight = (Weight)2;
|
||||
fst->AddArc(s1, arc); // arc from 1 to 2 with "1" out.
|
||||
}
|
||||
std::cout <<" printing before adding self-loops\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, ilabels, olabels, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
// So states 1 and 2 should have self-loops on.
|
||||
size_t num_extra = kaldi::Rand() % 5;
|
||||
vector<Label> extra_ilabels, extra_olabels;
|
||||
CreateNewSymbols(ilabels, num_extra, "in#", &extra_ilabels);
|
||||
CreateNewSymbols(olabels, num_extra, "out#", &extra_olabels);
|
||||
|
||||
AddSelfLoops(fst, extra_ilabels, extra_olabels);
|
||||
|
||||
assert(fst->NumArcs(0) == 1);
|
||||
assert(fst->NumArcs(1) == 1 + num_extra);
|
||||
assert(fst->NumArcs(2) == num_extra);
|
||||
|
||||
std::cout <<" printing after adding self-loops\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, ilabels, olabels, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
delete fst;
|
||||
delete ilabels;
|
||||
delete olabels;
|
||||
}
|
||||
|
||||
} // end namespace fst.
|
||||
|
||||
|
||||
int main() {
|
||||
for (int i = 0;i < 10;i++) { // run it multiple times; it's a randomized testing algorithm.
|
||||
fst::TestPreDeterminize<fst::StdArc>();
|
||||
}
|
||||
for (int i = 0;i < 5;i++) {
|
||||
fst::TestAddSelfLoops<fst::StdArc>();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// fstext/pre-determinize.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_PRE_DETERMINIZE_H_
|
||||
#define KALDI_FSTEXT_PRE_DETERMINIZE_H_
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
#include "base/kaldi-common.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
/* PreDeterminize inserts extra symbols on the input side of an FST as necessary to
|
||||
ensure that, after epsilon removal, it will be compactly determinizable by the
|
||||
determinize* algorithm. By compactly determinizable we mean that
|
||||
no original FST state is represented in more than one determinized state).
|
||||
|
||||
Caution: this code is now only used in testing.
|
||||
|
||||
The new symbols start from the value "first_new_symbol", which should be
|
||||
higher than the largest-numbered symbol currently in the FST. The new
|
||||
symbols added are put in the array syms_out, which should be empty at start.
|
||||
*/
|
||||
|
||||
template<class Arc, class Int>
|
||||
void PreDeterminize(MutableFst<Arc> *fst,
|
||||
typename Arc::Label first_new_symbol,
|
||||
std::vector<Int> *syms_out);
|
||||
|
||||
|
||||
/* CreateNewSymbols is a helper function used inside PreDeterminize, and is also useful
|
||||
when you need to add a number of extra symbols to a different vocabulary from the one
|
||||
modified by PreDeterminize. */
|
||||
|
||||
template<class Label>
|
||||
void CreateNewSymbols(SymbolTable *inputSymTable, int nSym,
|
||||
std::string prefix, std::vector<Label> *syms_out);
|
||||
|
||||
/** AddSelfLoops is a function you will probably want to use alongside PreDeterminize,
|
||||
to add self-loops to any FSTs that you compose on the left hand side of the one
|
||||
modified by PreDeterminize.
|
||||
|
||||
This function inserts loops with "special symbols" [e.g. \#0, \#1] into an FST.
|
||||
This is done at each final state and each state with non-epsilon output symbols on
|
||||
at least one arc out of it. This is to ensure that these symbols, when inserted into
|
||||
the input side of an FST we will compose with on the right, can "pass through" this
|
||||
FST.
|
||||
|
||||
At input, isyms and osyms must be vectors of the same size n, corresponding
|
||||
to symbols that currently do not exist in 'fst'. For each state in n that has
|
||||
non-epsilon symbols on the output side of arcs leaving it, or which is a final state,
|
||||
this function inserts n self-loops with unit weight and one of the n pairs
|
||||
of symbols on its input and output.
|
||||
*/
|
||||
template<class Arc>
|
||||
void AddSelfLoops(MutableFst<Arc> *fst, std::vector<typename Arc::Label> &isyms,
|
||||
std::vector<typename Arc::Label> &osyms);
|
||||
|
||||
|
||||
/* DeleteSymbols replaces any instances of symbols in the vector symsIn, appearing
|
||||
on the input side, with epsilon. */
|
||||
/* It returns the number of instances of symbols deleted. */
|
||||
template<class Arc>
|
||||
int64 DeleteISymbols(MutableFst<Arc> *fst, std::vector<typename Arc::Label> symsIn);
|
||||
|
||||
/* CreateSuperFinal takes an FST, and creates an equivalent FST with a single final
|
||||
state with no transitions out and unit final weight, by inserting epsilon transitions
|
||||
as necessary. */
|
||||
template<class Arc>
|
||||
typename Arc::StateId CreateSuperFinal(MutableFst<Arc> *fst);
|
||||
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
#include "fstext/pre-determinize-inl.h"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,173 @@
|
||||
// fstext/prune-special-inl.h
|
||||
|
||||
// Copyright 2014 Johns Hopkins University (Author: Daniel Povey)
|
||||
// Guoguo Chen
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_PRUNE_SPECIAL_INL_H_
|
||||
#define KALDI_FSTEXT_PRUNE_SPECIAL_INL_H_
|
||||
// Do not include this file directly. It is included by prune-special.h
|
||||
|
||||
#include "fstext/prune-special.h"
|
||||
#include "base/kaldi-error.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
/// This class is used to implement the function PruneSpecial.
|
||||
template<class Arc> class PruneSpecialClass {
|
||||
public:
|
||||
typedef typename Arc::StateId InputStateId;
|
||||
typedef typename Arc::StateId OutputStateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef typename Arc::Label Label;
|
||||
|
||||
PruneSpecialClass(const Fst<Arc> &ifst,
|
||||
VectorFst<Arc> *ofst,
|
||||
Weight beam,
|
||||
size_t max_states):
|
||||
ifst_(ifst), ofst_(ofst), beam_(beam), max_states_(max_states),
|
||||
best_weight_(Weight::Zero()) {
|
||||
KALDI_ASSERT(beam != Weight::One());
|
||||
KALDI_ASSERT(queue_.size() == 0);
|
||||
ofst_->DeleteStates(); // make sure it's empty.
|
||||
if (ifst_.Start() == kNoStateId)
|
||||
return;
|
||||
ofst_->SetStart(ProcessState(ifst_.Start(), Weight::One()));
|
||||
|
||||
while (!queue_.empty()) {
|
||||
Task task = queue_.top();
|
||||
queue_.pop();
|
||||
if (Done(task)) break;
|
||||
else ProcessTask(task);
|
||||
}
|
||||
Connect(ofst);
|
||||
if (beam_ != Weight::One())
|
||||
Prune(ofst, beam_);
|
||||
}
|
||||
|
||||
struct Task {
|
||||
InputStateId istate;
|
||||
OutputStateId ostate; // could be looked up; this is for speed.
|
||||
size_t position; // arc position, or -1 if final-prob.
|
||||
Weight weight;
|
||||
|
||||
Task(InputStateId istate, OutputStateId ostate, size_t position,
|
||||
Weight weight): istate(istate), ostate(ostate), position(position),
|
||||
weight(weight) { }
|
||||
bool operator < (const Task &other) const {
|
||||
return Compare(weight, other.weight) < 0;
|
||||
}
|
||||
};
|
||||
|
||||
bool Done(const Task &task) {
|
||||
if (beam_ != Weight::One() && best_weight_ != Weight::Zero() &&
|
||||
Compare(task.weight, Times(best_weight_, beam_)) < 0)
|
||||
return true;
|
||||
if (max_states_ > 0 &&
|
||||
static_cast<size_t>(ofst_->NumStates()) > max_states_)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// This function assumes "state" has not been seen before, so we need to
|
||||
// create a new output-state for it and add tasks. It returns the
|
||||
// output-state id. "weight" is the best cost from the start-state to this
|
||||
// state.
|
||||
inline OutputStateId ProcessState(InputStateId istate, const Weight &weight) {
|
||||
OutputStateId ostate = ofst_->AddState();
|
||||
state_map_[istate] = ostate;
|
||||
for (ArcIterator<Fst<Arc> > aiter(ifst_, istate); !aiter.Done();
|
||||
aiter.Next()) {
|
||||
const Arc &arc = aiter.Value();
|
||||
Task new_task(istate, ostate, aiter.Position(),
|
||||
Times(weight, arc.weight));
|
||||
KALDI_ASSERT(Compare(arc.weight, Weight::One()) != 1);
|
||||
queue_.push(new_task);
|
||||
}
|
||||
Weight final = ifst_.Final(istate);
|
||||
if (final != Weight::Zero()) {
|
||||
Task final_task(istate, ostate, static_cast<size_t>(-1),
|
||||
Times(weight, final));
|
||||
KALDI_ASSERT(Compare(final, Weight::One()) != 1);
|
||||
queue_.push(final_task);
|
||||
}
|
||||
return ostate;
|
||||
}
|
||||
|
||||
// Returns the output-state id corresponding to "istate". This assumes we are
|
||||
// processing a task corresponding to an arc to "istate", and the cost from
|
||||
// the start-state to this state is "weight". Since we process tasks in
|
||||
// order, if this is the first time we see this istate, then this is the best
|
||||
// cost from the start-state to this state, and it can be used in setting the
|
||||
// priority costs in ProcessState().
|
||||
inline OutputStateId GetOutputStateId(InputStateId istate,
|
||||
const Weight &weight) {
|
||||
typedef typename unordered_map<InputStateId, OutputStateId>::iterator IterType;
|
||||
IterType iter = state_map_.find(istate);
|
||||
if (iter == state_map_.end())
|
||||
return ProcessState(istate, weight);
|
||||
else
|
||||
return iter->second;
|
||||
}
|
||||
|
||||
void ProcessTask(const Task &task) {
|
||||
if (task.position == static_cast<size_t>(-1)) {
|
||||
ofst_->SetFinal(task.ostate, ifst_.Final(task.istate));
|
||||
if (best_weight_ == Weight::Zero())
|
||||
best_weight_ = task.weight; // best-path cost through FST, used for
|
||||
// beam-pruning.
|
||||
} else {
|
||||
ArcIterator<Fst<Arc> > aiter(ifst_, task.istate);
|
||||
aiter.Seek(task.position); // if we spend most of our time here, we may
|
||||
// need to store the arc in the Task.
|
||||
const Arc &arc = aiter.Value();
|
||||
InputStateId next_istate = arc.nextstate;
|
||||
OutputStateId next_ostate = GetOutputStateId(next_istate, task.weight);
|
||||
Arc oarc(arc.ilabel, arc.olabel, arc.weight, next_ostate);
|
||||
ofst_->AddArc(task.ostate, oarc);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const Fst<Arc> &ifst_;
|
||||
VectorFst<Arc> *ofst_;
|
||||
Weight beam_;
|
||||
size_t max_states_;
|
||||
|
||||
unordered_map<InputStateId, OutputStateId> state_map_;
|
||||
std::priority_queue<Task> queue_;
|
||||
Weight best_weight_; // if not Zero(), then we have now processed a successful path
|
||||
// through ifst_, and this is the weight.
|
||||
|
||||
};
|
||||
|
||||
template<class Arc>
|
||||
void PruneSpecial(const Fst<Arc> &ifst,
|
||||
VectorFst<Arc> *ofst,
|
||||
typename Arc::Weight beam,
|
||||
size_t max_states) {
|
||||
PruneSpecialClass<Arc> c(ifst, ofst, beam, max_states);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,79 @@
|
||||
// fstext/prune-special-test.cc
|
||||
|
||||
// Copyright 2014 Guoguo Chen
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#include "fstext/prune-special.h"
|
||||
#include "fstext/rand-fst.h"
|
||||
#include "fstext/fstext-utils.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
static void TestPruneSpecial() {
|
||||
typedef StdArc Arc;
|
||||
typedef Arc::Label Label;
|
||||
typedef Arc::StateId StateId;
|
||||
typedef Arc::Weight Weight;
|
||||
|
||||
RandFstOptions opts;
|
||||
opts.acyclic = false;
|
||||
VectorFst<Arc> *ifst = RandFst<StdArc>(opts);
|
||||
|
||||
float beam = 0.55;
|
||||
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*ifst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
// Do the special pruning.
|
||||
VectorFst<Arc> ofst1;
|
||||
PruneSpecial<StdArc>(*ifst, &ofst1, beam);
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(ofst1, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
// Do the normal pruning.
|
||||
VectorFst<Arc> ofst2;
|
||||
Prune(*ifst, &ofst2, beam);
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(ofst2, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
KALDI_ASSERT(RandEquivalent(ofst1, ofst2,
|
||||
5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/,
|
||||
100/*path length-- max?*/));
|
||||
|
||||
delete ifst;
|
||||
}
|
||||
|
||||
|
||||
} // namespace fst
|
||||
|
||||
int main() {
|
||||
kaldi::g_kaldi_verbose_level = 4;
|
||||
using namespace fst;
|
||||
for (int i = 0; i < 25; i++) {
|
||||
TestPruneSpecial();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// fstext/prune-special.h
|
||||
|
||||
// Copyright 2014 Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#ifndef KALDI_FSTEXT_PRUNE_SPECIAL_H_
|
||||
#define KALDI_FSTEXT_PRUNE_SPECIAL_H_
|
||||
|
||||
#include "fst/fstlib.h"
|
||||
#include "fstext/lattice-weight.h"
|
||||
#include "fstext/factor.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
|
||||
/**
|
||||
The function PruneSpecial is like the standard OpenFst function "prune",
|
||||
except it does not expand the entire "ifst"- this is useful for cases where
|
||||
ifst is an on-demand FST such as a ComposeFst and we don't want to visit
|
||||
it all. It supports pruning either to a specified beam (if beam is
|
||||
not One()), or to a specified max_states (if max_states is > 0). One of the
|
||||
two must be specified.
|
||||
|
||||
Requirements:
|
||||
- Costs must be non-negative (equivalently, weights must not be greater than One()).
|
||||
- There must be a Compare(a, b) function that compares two weights and returns (-1,0,1)
|
||||
if (a<b, a=b, a>b). We define this in Kaldi, for TropicalWeight, LogWeight (I think),
|
||||
and LatticeWeight... also CompactLatticeWeight, but we doubt that will be used here;
|
||||
better to use PruneCompactLattice().
|
||||
*/
|
||||
|
||||
template<class Arc>
|
||||
void PruneSpecial(const Fst<Arc> &ifst,
|
||||
VectorFst<Arc> *ofst,
|
||||
typename Arc::Weight beam,
|
||||
size_t max_states = 0);
|
||||
|
||||
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
#include "fstext/prune-special-inl.h"
|
||||
|
||||
#endif // KALDI_FSTEXT_PRUNE_SPECIAL_H_
|
||||
@@ -0,0 +1,81 @@
|
||||
// fstext/push-special-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#include "fstext/push-special.h"
|
||||
#include "fstext/rand-fst.h"
|
||||
#include "fstext/fstext-utils.h"
|
||||
#include "base/kaldi-math.h"
|
||||
|
||||
namespace fst
|
||||
{
|
||||
|
||||
|
||||
// Don't instantiate with log semiring, as RandEquivalent may fail.
|
||||
static void TestPushSpecial() {
|
||||
typedef StdArc Arc;
|
||||
typedef Arc::Label Label;
|
||||
typedef Arc::StateId StateId;
|
||||
typedef Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> *fst = RandFst<StdArc>();
|
||||
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
VectorFst<Arc> fst_copy(*fst);
|
||||
|
||||
float delta = kDelta;
|
||||
PushSpecial(&fst_copy, delta);
|
||||
|
||||
Weight min, max;
|
||||
float delta_dontcare = 0.1;
|
||||
IsStochasticFstInLog(fst_copy, delta_dontcare, &min, &max);
|
||||
// the per-state normalizers are allowed to deviate from the average by delta
|
||||
// up and down, so the difference from the min to max weight should be 2*delta
|
||||
// or less. We give it a bit of wiggle room (->2.5) due to numerical roundoff.
|
||||
|
||||
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(fst_copy, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
KALDI_LOG << "Min value is " << min.Value() << ", max value is " << max.Value();
|
||||
|
||||
// below, should be <= delta but different pieces of code compute this in this
|
||||
// part vs. push-special, so the roundoff may be different.
|
||||
KALDI_ASSERT(std::abs(min.Value() - max.Value()) <= 1.2 * delta);
|
||||
|
||||
KALDI_ASSERT(RandEquivalent(*fst, fst_copy,
|
||||
5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
delete fst;
|
||||
}
|
||||
|
||||
|
||||
} // namespace fst
|
||||
|
||||
int main() {
|
||||
kaldi::g_kaldi_verbose_level = 4;
|
||||
using namespace fst;
|
||||
for (int i = 0; i < 25; i++) {
|
||||
TestPushSpecial();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
// fstext/push-special.cc
|
||||
|
||||
// Copyright 2012 Johns Hopkins University (author: Daniel Povey)
|
||||
// 2012 Ehsan Variani, Pegah Ghahrmani
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fstext/push-special.h"
|
||||
#include "base/kaldi-error.h"
|
||||
#include "base/kaldi-math.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
/*
|
||||
|
||||
This algorithm was briefly described in "COMBINING FORWARD AND BACKWARD SEARCH
|
||||
IN DECODING" by Hannemann and Povey, ICASSP 2013,
|
||||
http://www.danielpovey.com/files/2013_icassp_pingpong.pdf
|
||||
|
||||
Below is the most relevant excerpt of the LaTeX source.
|
||||
|
||||
Real backoff language models represented as WFSTs (\cite{Mohri:08}) will not
|
||||
exactly sum to one because the backoff structure leads to duplicate paths for
|
||||
some word sequences. In fact, such language models cannot be pushed at all in
|
||||
the general case, because the total weight of the entire WFST may not be finite.
|
||||
For our language model reversal we need a suitable pushing operation that will
|
||||
always succeed.
|
||||
|
||||
Our solution is to require a modified pushing operation such that each state
|
||||
``sums to'' the same quantity.
|
||||
We were able to find an iterative algorithm that does this very efficiently in practice;
|
||||
it is based on the power method for finding the top eigenvalue of a matrix.
|
||||
Both for the math and the implementation, we find it more convenient to
|
||||
use the probability semiring, i.e. we represent the transition-probabilities
|
||||
as actual probabilities, not negative logs.
|
||||
Let the transitions be written as a sparse matrix $\mathbf{P}$,
|
||||
where $p_{ij}$ is the sum of all the probabilities of transitions between state $i$ and state $j$.
|
||||
As a special case, if $j$ is the initial state,
|
||||
then $p_{ij}$ is the final-probability of state $i$.
|
||||
In our method we find the dominant eigenvector $\mathbf{v}$ of the matrix $\mathbf{P}$,
|
||||
by starting from a random positive vector and iterating with the power method:
|
||||
each time we let $\mathbf{v} \leftarrow \mathbf{P} \mathbf{v}$
|
||||
and then renormalize the length of $\mathbf{v}$.
|
||||
It is convenient to renormalize $\mathbf{v}$ so that $v_I$ is 1,
|
||||
where $I$ is the initial state of the WFST\footnote{Note: in order to correctly
|
||||
deal with the case of linear WFSTs, which have different eigenvalues
|
||||
with the same magnitude but different complex phase,
|
||||
we modify the iteration to $\mathbf{v} \leftarrow \mathbf{P} \mathbf{v} + 0.1 \mathbf{v}$.}.
|
||||
This generally converges within several tens of iterations.
|
||||
At the end we have a vector $\mathbf{v}$ with $v_I = 1$, and a scalar $\lambda > 0$, such that
|
||||
\begin{equation}
|
||||
\lambda \mathbf{v} = \mathbf{P} \mathbf{v} . \label{eqn:lambdav}
|
||||
\end{equation}
|
||||
Suppose we compute a modified transition matrix $\mathbf{P}'$, by letting
|
||||
\begin{equation}
|
||||
p'_{ij} = p_{ij} v_j / v_i .
|
||||
\end{equation}
|
||||
Then it is easy to show each row of $\mathbf{P}'$ sums to $\lambda$:
|
||||
writing one element of Eq.~\ref{eqn:lambdav} as
|
||||
\begin{equation}
|
||||
\lambda v_i = \sum_j p_{ij} v_j,
|
||||
\end{equation}
|
||||
it easily follows that $\lambda = \sum_j p'_{ij}$.
|
||||
We need to perform a similar transformation on the transition-probabilities and
|
||||
final-probabilities of the WFST; the details are quite obvious, and the
|
||||
equivalence with the original WFST is easy to show. Our algorithm is in
|
||||
practice an order of magnitude faster than the more generic algorithm for
|
||||
conventional weight-pushing of \cite{Mohri:02}, when applied to cyclic WFSTs.
|
||||
|
||||
*/
|
||||
|
||||
class PushSpecialClass {
|
||||
typedef StdArc Arc;
|
||||
typedef Arc::Weight Weight;
|
||||
typedef Arc::StateId StateId;
|
||||
|
||||
public:
|
||||
// Everything happens in the initializer.
|
||||
PushSpecialClass(VectorFst<StdArc> *fst,
|
||||
float delta): fst_(fst) {
|
||||
num_states_ = fst_->NumStates();
|
||||
initial_state_ = fst_->Start();
|
||||
occ_.resize(num_states_, 1.0 / sqrt(num_states_)); // unit length
|
||||
|
||||
pred_.resize(num_states_);
|
||||
for (StateId s = 0; s < num_states_; s++) {
|
||||
for (ArcIterator<VectorFst<StdArc> > aiter(*fst, s);
|
||||
!aiter.Done(); aiter.Next()) {
|
||||
const Arc &arc = aiter.Value();
|
||||
StateId t = arc.nextstate;
|
||||
double weight = kaldi::Exp(-arc.weight.Value());
|
||||
pred_[t].push_back(std::make_pair(s, weight));
|
||||
}
|
||||
double final = kaldi::Exp(-fst_->Final(s).Value());
|
||||
if (final != 0.0)
|
||||
pred_[initial_state_].push_back(std::make_pair(s, final));
|
||||
}
|
||||
Iterate(delta);
|
||||
ModifyFst();
|
||||
}
|
||||
private:
|
||||
double TestAccuracy() { // returns the error (the difference
|
||||
// between the min and max weights).
|
||||
double min_sum = 0, max_sum = 0;
|
||||
for (StateId s = 0; s < num_states_; s++) {
|
||||
double sum = 0.0;
|
||||
for (ArcIterator<VectorFst<StdArc> > aiter(*fst_, s);
|
||||
!aiter.Done(); aiter.Next()) {
|
||||
const Arc &arc = aiter.Value();
|
||||
StateId t = arc.nextstate;
|
||||
sum += kaldi::Exp(-arc.weight.Value()) * occ_[t] / occ_[s];
|
||||
}
|
||||
sum += kaldi::Exp(-(fst_->Final(s).Value())) * occ_[initial_state_] / occ_[s];
|
||||
if (s == 0) {
|
||||
min_sum = sum;
|
||||
max_sum = sum;
|
||||
} else {
|
||||
min_sum = std::min(min_sum, sum);
|
||||
max_sum = std::max(max_sum, sum);
|
||||
}
|
||||
}
|
||||
KALDI_VLOG(4) << "min,max is " << min_sum << " " << max_sum;
|
||||
return kaldi::Log(max_sum / min_sum); // In FST world we'll actually
|
||||
// dealing with logs, so the log of the ratio is more suitable
|
||||
// to compare with delta (makes testing the algorithm easier).
|
||||
}
|
||||
|
||||
|
||||
void Iterate(float delta) {
|
||||
// This is like the power method to find the top eigenvalue of a matrix.
|
||||
// We limit it to 200 iters max, just in case something unanticipated
|
||||
// happens, but we should exit due to the "delta" thing, usually after
|
||||
// several tens of iterations.
|
||||
int iter, max_iter = 200;
|
||||
|
||||
for (iter = 0; iter < max_iter; iter++) {
|
||||
std::vector<double> new_occ(num_states_);
|
||||
// We initialize new_occ to 0.1 * occ. A simpler algorithm would
|
||||
// initialize them to zero, so it's like the pure power method. This is
|
||||
// like the power method on (M + 0.1 I), and we do it this way to avoid a
|
||||
// problem we encountered with certain very simple linear FSTs where the
|
||||
// eigenvalues of the weight matrix (including negative and imaginary
|
||||
// ones) all have the same magnitude.
|
||||
for (int i = 0; i < num_states_; i++)
|
||||
new_occ[i] = 0.1 * occ_[i];
|
||||
|
||||
for (int i = 0; i < num_states_; i++) {
|
||||
std::vector<std::pair<StateId, double> >::const_iterator iter,
|
||||
end = pred_[i].end();
|
||||
for (iter = pred_[i].begin(); iter != end; ++iter) {
|
||||
StateId j = iter->first;
|
||||
double p = iter->second;
|
||||
new_occ[j] += occ_[i] * p;
|
||||
}
|
||||
}
|
||||
double sumsq = 0.0;
|
||||
for (int i = 0; i < num_states_; i++) sumsq += new_occ[i] * new_occ[i];
|
||||
lambda_ = std::sqrt(sumsq);
|
||||
double inv_lambda = 1.0 / lambda_;
|
||||
for (int i = 0; i < num_states_; i++) occ_[i] = new_occ[i] * inv_lambda;
|
||||
KALDI_VLOG(4) << "Lambda is " << lambda_;
|
||||
if (iter % 5 == 0 && iter > 0 && TestAccuracy() <= delta) {
|
||||
KALDI_VLOG(3) << "Weight-pushing converged after " << iter
|
||||
<< " iterations.";
|
||||
return;
|
||||
}
|
||||
}
|
||||
KALDI_WARN << "push-special: finished " << iter
|
||||
<< " iterations without converging. Output will be inaccurate.";
|
||||
}
|
||||
|
||||
|
||||
// Modifies the FST weights and the final-prob to take account of these potentials.
|
||||
void ModifyFst() {
|
||||
// First get the potentials as negative-logs, like the values
|
||||
// in the FST.
|
||||
for (StateId s = 0; s < num_states_; s++) {
|
||||
occ_[s] = -kaldi::Log(occ_[s]);
|
||||
if (KALDI_ISNAN(occ_[s]) || KALDI_ISINF(occ_[s]))
|
||||
KALDI_WARN << "NaN or inf found: " << occ_[s];
|
||||
}
|
||||
for (StateId s = 0; s < num_states_; s++) {
|
||||
for (MutableArcIterator<VectorFst<StdArc> > aiter(fst_, s);
|
||||
!aiter.Done(); aiter.Next()) {
|
||||
Arc arc = aiter.Value();
|
||||
StateId t = arc.nextstate;
|
||||
arc.weight = Weight(arc.weight.Value() + occ_[t] - occ_[s]);
|
||||
aiter.SetValue(arc);
|
||||
}
|
||||
fst_->SetFinal(s, Times(fst_->Final(s).Value(),
|
||||
Weight(occ_[initial_state_] - occ_[s])));
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
StateId num_states_;
|
||||
StateId initial_state_;
|
||||
std::vector<double> occ_; // the top eigenvector of (matrix of weights) transposed.
|
||||
double lambda_; // our current estimate of the top eigenvalue.
|
||||
|
||||
std::vector<std::vector<std::pair<StateId, double> > > pred_; // List of transitions
|
||||
// into each state. For the start state, this list consists of the list of
|
||||
// states with final-probs, each with their final prob.
|
||||
|
||||
VectorFst<StdArc> *fst_;
|
||||
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
void PushSpecial(VectorFst<StdArc> *fst, float delta) {
|
||||
if (fst->NumStates() > 0)
|
||||
PushSpecialClass c(fst, delta); // all the work
|
||||
// gets done in the initializer.
|
||||
}
|
||||
|
||||
|
||||
} // end namespace fst.
|
||||
|
||||
|
||||
/*
|
||||
Note: in testing an earlier, simpler
|
||||
version of this method (without the 0.1 * old_occ) we had a problem with the following FST.
|
||||
0 2 3 3 0
|
||||
1 3 1 4 0.5
|
||||
2 1 0 0 0.5
|
||||
3 0.25
|
||||
|
||||
Corresponds to the following matrix [or maybe its transpose, doesn't matter
|
||||
probably]
|
||||
|
||||
a=exp(-0.5)
|
||||
b=exp(-0.25)
|
||||
M = [ 0 1 0 0
|
||||
0 0 a 0
|
||||
0 0 0 a
|
||||
b 0 0 0 ]
|
||||
|
||||
eigs(M)
|
||||
eigs(M)
|
||||
|
||||
ans =
|
||||
|
||||
-0.0000 - 0.7316i
|
||||
-0.0000 + 0.7316i
|
||||
0.7316
|
||||
-0.7316
|
||||
|
||||
OK, so the issue appears to be that all the eigenvalues of this matrix
|
||||
have the same magnitude. The solution is to work with the eigenvalues
|
||||
of M + alpha I, for some small alpha such as 0.1 (so as not to slow down
|
||||
convergence in the normal case).
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,42 @@
|
||||
// fstext/push-special.h
|
||||
|
||||
// Copyright 2012-2015 Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_PUSH_SPECIAL_H_
|
||||
#define KALDI_FSTEXT_PUSH_SPECIAL_H_
|
||||
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
#include "util/const-integer-set.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
/*
|
||||
This function does weight-pushing, in the log semiring,
|
||||
but in a special way, such that any "leftover weight" after pushing
|
||||
gets distributed evenly along the FST, and doesn't end up either
|
||||
at the start or at the end. Basically it pushes the weights such
|
||||
that the total weight of each state (i.e. the sum of the arc
|
||||
probabilities plus the final-prob) is the same for all states.
|
||||
*/
|
||||
void PushSpecial(VectorFst<StdArc> *fst,
|
||||
float delta = kDelta);
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,162 @@
|
||||
// fstext/rand-fst.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_RAND_FST_H_
|
||||
#define KALDI_FSTEXT_RAND_FST_H_
|
||||
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
#include "base/kaldi-math.h"
|
||||
|
||||
namespace fst {
|
||||
|
||||
// Note: all weights are constructed from nonnegative floats.
|
||||
// (so no "negative costs").
|
||||
struct RandFstOptions {
|
||||
size_t n_syms;
|
||||
size_t n_states;
|
||||
size_t n_arcs;
|
||||
size_t n_final;
|
||||
bool allow_empty;
|
||||
bool acyclic;
|
||||
float weight_multiplier;
|
||||
RandFstOptions() { // Initializes the options randomly.
|
||||
n_syms = 2 + kaldi::Rand() % 5;
|
||||
n_states = 3 + kaldi::Rand() % 10;
|
||||
n_arcs = 5 + kaldi::Rand() % 30;
|
||||
n_final = 1 + kaldi::Rand()%3;
|
||||
allow_empty = true;
|
||||
acyclic = false;
|
||||
weight_multiplier = 0.25;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/// Returns a random FST. Useful for randomized algorithm testing.
|
||||
/// Only works if weight can be constructed from float.
|
||||
template<class Arc> VectorFst<Arc>* RandFst(RandFstOptions opts = RandFstOptions() ) {
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> *fst = new VectorFst<Arc>();
|
||||
|
||||
start:
|
||||
|
||||
// Create states.
|
||||
std::vector<StateId> all_states;
|
||||
for (size_t i = 0;i < (size_t)opts.n_states;i++) {
|
||||
StateId this_state = fst->AddState();
|
||||
if (i == 0) fst->SetStart(i);
|
||||
all_states.push_back(this_state);
|
||||
}
|
||||
// Set final states.
|
||||
for (size_t j = 0;j < (size_t)opts.n_final;j++) {
|
||||
StateId id = all_states[kaldi::Rand() % opts.n_states];
|
||||
Weight weight = (Weight)(opts.weight_multiplier*(kaldi::Rand() % 5));
|
||||
fst->SetFinal(id, weight);
|
||||
}
|
||||
// Create arcs.
|
||||
for (size_t i = 0;i < (size_t)opts.n_arcs;i++) {
|
||||
Arc a;
|
||||
StateId start_state;
|
||||
if(!opts.acyclic) { // no restriction on arcs.
|
||||
start_state = all_states[kaldi::Rand() % opts.n_states];
|
||||
a.nextstate = all_states[kaldi::Rand() % opts.n_states];
|
||||
} else {
|
||||
start_state = all_states[kaldi::Rand() % (opts.n_states-1)];
|
||||
a.nextstate = start_state + 1 + (kaldi::Rand() % (opts.n_states-start_state-1));
|
||||
}
|
||||
a.ilabel = kaldi::Rand() % opts.n_syms;
|
||||
a.olabel = kaldi::Rand() % opts.n_syms; // same input+output vocab.
|
||||
a.weight = (Weight) (opts.weight_multiplier*(kaldi::Rand() % 4));
|
||||
|
||||
fst->AddArc(start_state, a);
|
||||
}
|
||||
|
||||
// Trim resulting FST.
|
||||
Connect(fst);
|
||||
if (opts.acyclic)
|
||||
assert(fst->Properties(kAcyclic, true) & kAcyclic);
|
||||
if (fst->Start() == kNoStateId && !opts.allow_empty) {
|
||||
goto start;
|
||||
}
|
||||
return fst;
|
||||
}
|
||||
|
||||
|
||||
/// Returns a random FST. Useful for randomized algorithm testing.
|
||||
/// Only works if weight can be constructed from a pair of floats
|
||||
template<class Arc> VectorFst<Arc>* RandPairFst(RandFstOptions opts = RandFstOptions() ) {
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> *fst = new VectorFst<Arc>();
|
||||
|
||||
start:
|
||||
|
||||
// Create states.
|
||||
std::vector<StateId> all_states;
|
||||
for (size_t i = 0;i < (size_t)opts.n_states;i++) {
|
||||
StateId this_state = fst->AddState();
|
||||
if (i == 0) fst->SetStart(i);
|
||||
all_states.push_back(this_state);
|
||||
}
|
||||
// Set final states.
|
||||
for (size_t j = 0; j < (size_t)opts.n_final;j++) {
|
||||
StateId id = all_states[kaldi::Rand() % opts.n_states];
|
||||
Weight weight (opts.weight_multiplier*(kaldi::Rand() % 5), opts.weight_multiplier*(kaldi::Rand() % 5));
|
||||
fst->SetFinal(id, weight);
|
||||
}
|
||||
// Create arcs.
|
||||
for (size_t i = 0;i < (size_t)opts.n_arcs;i++) {
|
||||
Arc a;
|
||||
StateId start_state;
|
||||
if(!opts.acyclic) { // no restriction on arcs.
|
||||
start_state = all_states[kaldi::Rand() % opts.n_states];
|
||||
a.nextstate = all_states[kaldi::Rand() % opts.n_states];
|
||||
} else {
|
||||
start_state = all_states[kaldi::Rand() % (opts.n_states-1)];
|
||||
a.nextstate = start_state + 1 + (kaldi::Rand() % (opts.n_states-start_state-1));
|
||||
}
|
||||
a.ilabel = kaldi::Rand() % opts.n_syms;
|
||||
a.olabel = kaldi::Rand() % opts.n_syms; // same input+output vocab.
|
||||
a.weight = Weight (opts.weight_multiplier*(kaldi::Rand() % 4), opts.weight_multiplier*(kaldi::Rand() % 4));
|
||||
|
||||
fst->AddArc(start_state, a);
|
||||
}
|
||||
|
||||
// Trim resulting FST.
|
||||
Connect(fst);
|
||||
if (opts.acyclic)
|
||||
assert(fst->Properties(kAcyclic, true) & kAcyclic);
|
||||
if (fst->Start() == kNoStateId && !opts.allow_empty) {
|
||||
goto start;
|
||||
}
|
||||
return fst;
|
||||
}
|
||||
|
||||
|
||||
} // end namespace fst.
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,321 @@
|
||||
// fstext/remove-eps-local-inl.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2014 Johns Hopkins University (author: Daniel Povey
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_REMOVE_EPS_LOCAL_INL_H_
|
||||
#define KALDI_FSTEXT_REMOVE_EPS_LOCAL_INL_H_
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
template<class Weight>
|
||||
struct ReweightPlusDefault {
|
||||
inline Weight operator () (const Weight &a, const Weight &b) {
|
||||
return Plus(a, b);
|
||||
}
|
||||
};
|
||||
|
||||
struct ReweightPlusLogArc {
|
||||
inline TropicalWeight operator () (const TropicalWeight &a,
|
||||
const TropicalWeight &b) {
|
||||
LogWeight a_log(a.Value()), b_log(b.Value());
|
||||
return TropicalWeight(Plus(a_log, b_log).Value());
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
template<class Arc, class ReweightPlus = ReweightPlusDefault<typename Arc::Weight> >
|
||||
class RemoveEpsLocalClass {
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
public:
|
||||
RemoveEpsLocalClass(MutableFst<Arc> *fst):
|
||||
fst_(fst) {
|
||||
if (fst_->Start() == kNoStateId) return; // empty.
|
||||
non_coacc_state_ = fst_->AddState();
|
||||
InitNumArcs();
|
||||
StateId num_states = fst_->NumStates();
|
||||
for (StateId s = 0; s < num_states; s++)
|
||||
for (size_t pos = 0; pos < fst_->NumArcs(s); pos++)
|
||||
RemoveEps(s, pos);
|
||||
assert(CheckNumArcs());
|
||||
Connect(fst); // remove inaccessible states.
|
||||
}
|
||||
private:
|
||||
MutableFst<Arc> *fst_;
|
||||
StateId non_coacc_state_; // use this to delete arcs: make it nextstate
|
||||
std::vector<StateId> num_arcs_in_; // The number of arcs into the state, plus one
|
||||
// if it's the start state.
|
||||
std::vector<StateId> num_arcs_out_; // The number of arcs out of the state, plus
|
||||
// one if it's a final state.
|
||||
ReweightPlus reweight_plus_;
|
||||
|
||||
bool CanCombineArcs(const Arc &a, const Arc &b, Arc *c) {
|
||||
if (a.ilabel != 0 && b.ilabel != 0) return false;
|
||||
if (a.olabel != 0 && b.olabel != 0) return false;
|
||||
c->weight = Times(a.weight, b.weight);
|
||||
c->ilabel = (a.ilabel != 0 ? a.ilabel : b.ilabel);
|
||||
c->olabel = (a.olabel != 0 ? a.olabel : b.olabel);
|
||||
c->nextstate = b.nextstate;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool CanCombineFinal(const Arc &a, Weight final_prob, Weight *final_prob_out) {
|
||||
if (a.ilabel != 0 || a.olabel != 0) return false;
|
||||
else {
|
||||
*final_prob_out = Times(a.weight, final_prob);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
void InitNumArcs() { // init num transitions in/out of each state.
|
||||
StateId num_states = fst_->NumStates();
|
||||
num_arcs_in_.resize(num_states);
|
||||
num_arcs_out_.resize(num_states);
|
||||
num_arcs_in_[fst_->Start()]++; // count start as trans in.
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
if (fst_->Final(s) != Weight::Zero())
|
||||
num_arcs_out_[s]++; // count final as transition.
|
||||
for (ArcIterator<MutableFst<Arc> > aiter(*fst_, s); !aiter.Done(); aiter.Next()) {
|
||||
num_arcs_in_[aiter.Value().nextstate]++;
|
||||
num_arcs_out_[s]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool CheckNumArcs() { // check num arcs in/out of each state, at end. Debug.
|
||||
num_arcs_in_[fst_->Start()]--; // count start as trans in.
|
||||
StateId num_states = fst_->NumStates();
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
if (s == non_coacc_state_) continue;
|
||||
if (fst_->Final(s) != Weight::Zero())
|
||||
num_arcs_out_[s]--; // count final as transition.
|
||||
for (ArcIterator<MutableFst<Arc> > aiter(*fst_, s); !aiter.Done(); aiter.Next()) {
|
||||
if (aiter.Value().nextstate == non_coacc_state_) continue;
|
||||
num_arcs_in_[aiter.Value().nextstate]--;
|
||||
num_arcs_out_[s]--;
|
||||
}
|
||||
}
|
||||
for (StateId s = 0; s < num_states; s++) {
|
||||
assert(num_arcs_in_[s] == 0);
|
||||
assert(num_arcs_out_[s] == 0);
|
||||
}
|
||||
return true; // always does this. so we can assert it w/o warnings.
|
||||
}
|
||||
|
||||
inline void GetArc(StateId s, size_t pos, Arc *arc) const {
|
||||
ArcIterator<MutableFst<Arc> > aiter(*fst_, s);
|
||||
aiter.Seek(pos);
|
||||
*arc = aiter.Value();
|
||||
}
|
||||
|
||||
inline void SetArc(StateId s, size_t pos, const Arc &arc) {
|
||||
MutableArcIterator<MutableFst<Arc> > aiter(fst_, s);
|
||||
aiter.Seek(pos);
|
||||
aiter.SetValue(arc);
|
||||
}
|
||||
|
||||
|
||||
void Reweight(StateId s, size_t pos, Weight reweight) {
|
||||
// Reweight is called from RemoveEpsPattern1; it is a step we
|
||||
// do to preserve stochasticity. This function multiplies the
|
||||
// arc at (s, pos) by reweight and divides all the arcs [+final-prob]
|
||||
// out of the next state by the same. This is only valid if
|
||||
// the next state has only one arc in and is not the start state.
|
||||
assert(reweight != Weight::Zero());
|
||||
MutableArcIterator<MutableFst<Arc> > aiter(fst_, s);
|
||||
aiter.Seek(pos);
|
||||
Arc arc = aiter.Value();
|
||||
assert(num_arcs_in_[arc.nextstate] == 1);
|
||||
arc.weight = Times(arc.weight, reweight);
|
||||
aiter.SetValue(arc);
|
||||
|
||||
for (MutableArcIterator<MutableFst<Arc> > aiter_next(fst_, arc.nextstate);
|
||||
!aiter_next.Done();
|
||||
aiter_next.Next()) {
|
||||
Arc nextarc = aiter_next.Value();
|
||||
if (nextarc.nextstate != non_coacc_state_) {
|
||||
nextarc.weight = Divide(nextarc.weight, reweight, DIVIDE_LEFT);
|
||||
aiter_next.SetValue(nextarc);
|
||||
}
|
||||
}
|
||||
Weight final = fst_->Final(arc.nextstate);
|
||||
if (final != Weight::Zero()) {
|
||||
fst_->SetFinal(arc.nextstate, Divide(final, reweight, DIVIDE_LEFT));
|
||||
}
|
||||
}
|
||||
|
||||
// RemoveEpsPattern1 applies where this arc, which is not a
|
||||
// self-loop, enters a state which has only one input transition
|
||||
// [and is not the start state], and has multiple output
|
||||
// transitions [counting being the final-state as a final-transition].
|
||||
|
||||
void RemoveEpsPattern1(StateId s, size_t pos, Arc arc) {
|
||||
const StateId nextstate = arc.nextstate;
|
||||
Weight total_removed = Weight::Zero(),
|
||||
total_kept = Weight::Zero(); // totals out of nextstate.
|
||||
std::vector<Arc> arcs_to_add; // to add to state s.
|
||||
for (MutableArcIterator<MutableFst<Arc> > aiter_next(fst_, nextstate);
|
||||
!aiter_next.Done();
|
||||
aiter_next.Next()) {
|
||||
Arc nextarc = aiter_next.Value();
|
||||
if (nextarc.nextstate == non_coacc_state_) continue; // deleted.
|
||||
Arc combined;
|
||||
if (CanCombineArcs(arc, nextarc, &combined)) {
|
||||
total_removed = reweight_plus_(total_removed, nextarc.weight);
|
||||
num_arcs_out_[nextstate]--;
|
||||
num_arcs_in_[nextarc.nextstate]--;
|
||||
nextarc.nextstate = non_coacc_state_;
|
||||
aiter_next.SetValue(nextarc);
|
||||
arcs_to_add.push_back(combined);
|
||||
} else {
|
||||
total_kept = reweight_plus_(total_kept, nextarc.weight);
|
||||
}
|
||||
}
|
||||
|
||||
{ // now final-state.
|
||||
Weight next_final = fst_->Final(nextstate);
|
||||
if (next_final != Weight::Zero()) {
|
||||
Weight new_final;
|
||||
if (CanCombineFinal(arc, next_final, &new_final)) {
|
||||
total_removed = reweight_plus_(total_removed, next_final);
|
||||
if (fst_->Final(s) == Weight::Zero())
|
||||
num_arcs_out_[s]++; // final is counted as arc.
|
||||
fst_->SetFinal(s, Plus(fst_->Final(s), new_final));
|
||||
num_arcs_out_[nextstate]--;
|
||||
fst_->SetFinal(nextstate, Weight::Zero());
|
||||
} else {
|
||||
total_kept = reweight_plus_(total_kept, next_final);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (total_removed != Weight::Zero()) { // did something...
|
||||
if (total_kept == Weight::Zero()) { // removed everything: remove arc.
|
||||
num_arcs_out_[s]--;
|
||||
num_arcs_in_[arc.nextstate]--;
|
||||
arc.nextstate = non_coacc_state_;
|
||||
SetArc(s, pos, arc);
|
||||
} else {
|
||||
// Have to reweight.
|
||||
Weight total = reweight_plus_(total_removed, total_kept);
|
||||
Weight reweight = Divide(total_kept, total, DIVIDE_LEFT); // <=1
|
||||
Reweight(s, pos, reweight);
|
||||
}
|
||||
}
|
||||
// Now add the arcs we were going to add.
|
||||
for (size_t i = 0; i < arcs_to_add.size(); i++) {
|
||||
num_arcs_out_[s]++;
|
||||
num_arcs_in_[arcs_to_add[i].nextstate]++;
|
||||
fst_->AddArc(s, arcs_to_add[i]);
|
||||
}
|
||||
}
|
||||
|
||||
void RemoveEpsPattern2(StateId s, size_t pos, Arc arc) {
|
||||
|
||||
// Pattern 2 is where "nextstate" has only one arc out, counting
|
||||
// being-the-final-state as an arc, but possibly multiple arcs in.
|
||||
// Also, nextstate != s.
|
||||
|
||||
const StateId nextstate = arc.nextstate;
|
||||
bool can_delete_next = (num_arcs_in_[nextstate] == 1); // if
|
||||
// we combine, can delete the corresponding out-arc/final-prob
|
||||
// of nextstate.
|
||||
bool delete_arc = false; // set to true if this arc to be deleted.
|
||||
|
||||
Weight next_final = fst_->Final(arc.nextstate);
|
||||
if (next_final != Weight::Zero()) { // nextstate has no actual arcs out, only final-prob.
|
||||
Weight new_final;
|
||||
if (CanCombineFinal(arc, next_final, &new_final)) {
|
||||
if (fst_->Final(s) == Weight::Zero())
|
||||
num_arcs_out_[s]++; // final is counted as arc.
|
||||
fst_->SetFinal(s, Plus(fst_->Final(s), new_final));
|
||||
delete_arc = true; // will delete "arc".
|
||||
if (can_delete_next) {
|
||||
num_arcs_out_[nextstate]--;
|
||||
fst_->SetFinal(nextstate, Weight::Zero());
|
||||
}
|
||||
}
|
||||
} else { // has an arc but no final prob.
|
||||
MutableArcIterator<MutableFst<Arc> > aiter_next(fst_, nextstate);
|
||||
assert(!aiter_next.Done());
|
||||
while (aiter_next.Value().nextstate == non_coacc_state_) {
|
||||
aiter_next.Next();
|
||||
assert(!aiter_next.Done());
|
||||
}
|
||||
// now aiter_next points to a real arc out of nextstate.
|
||||
Arc nextarc = aiter_next.Value();
|
||||
Arc combined;
|
||||
if (CanCombineArcs(arc, nextarc, &combined)) {
|
||||
delete_arc = true;
|
||||
if (can_delete_next) { // do it before we invalidate iterators
|
||||
num_arcs_out_[nextstate]--;
|
||||
num_arcs_in_[nextarc.nextstate]--;
|
||||
nextarc.nextstate = non_coacc_state_;
|
||||
aiter_next.SetValue(nextarc);
|
||||
}
|
||||
num_arcs_out_[s]++;
|
||||
num_arcs_in_[combined.nextstate]++;
|
||||
fst_->AddArc(s, combined);
|
||||
}
|
||||
}
|
||||
if (delete_arc) {
|
||||
num_arcs_out_[s]--;
|
||||
num_arcs_in_[nextstate]--;
|
||||
arc.nextstate = non_coacc_state_;
|
||||
SetArc(s, pos, arc);
|
||||
}
|
||||
}
|
||||
|
||||
void RemoveEps(StateId s, size_t pos) {
|
||||
// Tries to do local epsilon-removal for arc sequences starting with this arc
|
||||
Arc arc;
|
||||
GetArc(s, pos, &arc);
|
||||
StateId nextstate = arc.nextstate;
|
||||
if (nextstate == non_coacc_state_) return; // deleted arc.
|
||||
if (nextstate == s) return; // don't handle self-loops: too complex.
|
||||
|
||||
if (num_arcs_in_[nextstate] == 1 && num_arcs_out_[nextstate] > 1) {
|
||||
RemoveEpsPattern1(s, pos, arc);
|
||||
} else if (num_arcs_out_[nextstate] == 1) {
|
||||
RemoveEpsPattern2(s, pos, arc);
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
||||
template<class Arc>
|
||||
void RemoveEpsLocal(MutableFst<Arc> *fst) {
|
||||
RemoveEpsLocalClass<Arc> c(fst); // work gets done in initializer.
|
||||
}
|
||||
|
||||
|
||||
void RemoveEpsLocalSpecial(MutableFst<StdArc> *fst) {
|
||||
// work gets done in initializer.
|
||||
RemoveEpsLocalClass<StdArc, ReweightPlusLogArc> c(fst);
|
||||
}
|
||||
|
||||
} // end namespace fst.
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,178 @@
|
||||
// fstext/remove-eps-local-test.cc
|
||||
|
||||
// Copyright 2009-2012 Microsoft Corporation Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#include "fstext/remove-eps-local.h"
|
||||
#include "fstext/fstext-utils.h"
|
||||
#include "fstext/fst-test-utils.h"
|
||||
#include "base/kaldi-math.h"
|
||||
|
||||
|
||||
namespace fst
|
||||
{
|
||||
using std::vector;
|
||||
using std::cout;
|
||||
|
||||
// Don't instantiate with log semiring, as RandEquivalent may fail.
|
||||
template<class Arc> static void TestRemoveEpsLocal() {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> fst;
|
||||
int n_syms = 2 + kaldi::Rand() % 5, n_arcs = 5 + kaldi::Rand() % 30, n_final = 1 + kaldi::Rand()%10;
|
||||
|
||||
SymbolTable symtab("my-symbol-table"), *sptr = &symtab;
|
||||
|
||||
vector<Label> all_syms; // including epsilon.
|
||||
// Put symbols in the symbol table from 1..n_syms-1.
|
||||
for (size_t i = 0;i < (size_t)n_syms;i++) {
|
||||
std::stringstream ss;
|
||||
if (i == 0) ss << "<eps>";
|
||||
else ss<<i;
|
||||
Label cur_lab = sptr->AddSymbol(ss.str());
|
||||
assert(cur_lab == (Label)i);
|
||||
all_syms.push_back(cur_lab);
|
||||
}
|
||||
assert(all_syms[0] == 0);
|
||||
|
||||
fst.AddState();
|
||||
int cur_num_states = 1;
|
||||
for (int i = 0; i < n_arcs; i++) {
|
||||
StateId src_state = kaldi::Rand() % cur_num_states;
|
||||
StateId dst_state;
|
||||
if (kaldi::RandUniform() < 0.1) dst_state = kaldi::Rand() % cur_num_states;
|
||||
else {
|
||||
dst_state = cur_num_states++; fst.AddState();
|
||||
}
|
||||
Arc arc;
|
||||
if (kaldi::RandUniform() < 0.3) arc.ilabel = all_syms[kaldi::Rand()%all_syms.size()];
|
||||
else arc.ilabel = 0;
|
||||
if (kaldi::RandUniform() < 0.3) arc.olabel = all_syms[kaldi::Rand()%all_syms.size()];
|
||||
else arc.olabel = 0;
|
||||
arc.weight = (Weight) (0 + 0.1*(kaldi::Rand() % 5));
|
||||
arc.nextstate = dst_state;
|
||||
fst.AddArc(src_state, arc);
|
||||
}
|
||||
for (int i = 0; i < n_final; i++) {
|
||||
fst.SetFinal(kaldi::Rand() % cur_num_states, (Weight) (0 + 0.1*(kaldi::Rand() % 5)));
|
||||
}
|
||||
|
||||
if (kaldi::RandUniform() < 0.8) fst.SetStart(0); // usually leads to nicer examples.
|
||||
else fst.SetStart(kaldi::Rand() % cur_num_states);
|
||||
|
||||
Connect(&fst);
|
||||
if (fst.Start() == kNoStateId) return; // "Connect" made it empty.
|
||||
|
||||
std::cout <<" printing after trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
VectorFst<Arc> fst_copy1(fst);
|
||||
|
||||
|
||||
RemoveEpsLocal(&fst_copy1);
|
||||
|
||||
|
||||
|
||||
{
|
||||
std::cout << "copy1 = \n";
|
||||
FstPrinter<Arc> fstprinter(fst_copy1, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
int num_states_0 = fst.NumStates();
|
||||
int num_states_1 = fst_copy1.NumStates();
|
||||
|
||||
|
||||
std::cout << "Number of states 0 = "<<num_states_0<<", 1 = "<<num_states_1<<'\n';
|
||||
|
||||
assert(RandEquivalent(fst, fst_copy1, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
}
|
||||
|
||||
|
||||
static void TestRemoveEpsLocalSpecial() {
|
||||
// test that RemoveEpsLocalSpecial preserves equivalence in tropical while
|
||||
// maintaining stochasticity in log.
|
||||
typedef VectorFst<LogArc> Fst;
|
||||
typedef LogArc::Weight Weight;
|
||||
typedef LogArc::StateId StateId;
|
||||
typedef LogArc Arc;
|
||||
VectorFst<LogArc> *logfst = RandFst<LogArc>();
|
||||
|
||||
{ // Make the FST stochastic.
|
||||
for (StateId s = 0; s < logfst->NumStates(); s++) {
|
||||
Weight w = logfst->Final(s);
|
||||
for (ArcIterator<Fst> aiter(*logfst, s); !aiter.Done(); aiter.Next()) {
|
||||
w = Plus(w, aiter.Value().weight);
|
||||
}
|
||||
if (w != Weight::Zero()) {
|
||||
logfst->SetFinal(s, Divide(logfst->Final(s), w, DIVIDE_ANY));
|
||||
for (MutableArcIterator<Fst> aiter(logfst, s); !aiter.Done(); aiter.Next()) {
|
||||
Arc a = aiter.Value();
|
||||
a.weight = Divide(a.weight, w, DIVIDE_ANY);
|
||||
aiter.SetValue(a);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#ifndef _MSC_VER
|
||||
assert(IsStochasticFst(*logfst, kDelta*10));
|
||||
#endif
|
||||
{
|
||||
std::cout << "logfst = \n";
|
||||
FstPrinter<LogArc> fstprinter(*logfst, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
VectorFst<StdArc> fst;
|
||||
Cast(*logfst, &fst);
|
||||
VectorFst<StdArc> fst_copy(fst);
|
||||
RemoveEpsLocalSpecial(&fst); // removes eps in std-arc but keep stochastic in log-arc
|
||||
// make sure equivalent.
|
||||
assert(RandEquivalent(fst, fst_copy, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
VectorFst<LogArc> logfst2;
|
||||
Cast(fst, &logfst2);
|
||||
|
||||
{
|
||||
std::cout << "logfst2 = \n";
|
||||
FstPrinter<LogArc> fstprinter(logfst2, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
if (ApproxEqual(ShortestDistance(*logfst), ShortestDistance(logfst2))) {
|
||||
// make sure we preserved stochasticity in cases where doing so was
|
||||
// possible... if the log-semiring total weight changed, then it is
|
||||
// not possible so don't assert this.
|
||||
assert(IsStochasticFst(logfst2, kDelta*10));
|
||||
}
|
||||
delete logfst;
|
||||
}
|
||||
|
||||
} // namespace fst
|
||||
|
||||
int main() {
|
||||
using namespace fst;
|
||||
for (int i = 0; i < 10; i++) {
|
||||
TestRemoveEpsLocal<fst::StdArc>();
|
||||
TestRemoveEpsLocalSpecial();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// fstext/remove-eps-local.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2014 Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_REMOVE_EPS_LOCAL_H_
|
||||
#define KALDI_FSTEXT_REMOVE_EPS_LOCAL_H_
|
||||
|
||||
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
/// RemoveEpsLocal remove some (but not necessarily all) epsilons in an FST,
|
||||
/// using an algorithm that is guaranteed to never increase the number of arcs
|
||||
/// in the FST (and will also never increase the number of states). The
|
||||
/// algorithm is not optimal but is reasonably clever. It does not just remove
|
||||
/// epsilon arcs;it also combines pairs of input-epsilon and output-epsilon arcs
|
||||
/// into one.
|
||||
/// The algorithm preserves equivalence and stochasticity in the given semiring.
|
||||
/// If you want to preserve stochasticity in a different semiring (e.g. log),
|
||||
/// then use RemoveEpsLocalSpecial, which only works for StdArc but which
|
||||
/// preserves stochasticity, where possible (*) in the LogArc sense. The reason that we can't
|
||||
/// just cast to a different semiring is that in that case we would no longer
|
||||
/// be able to guarantee equivalence in the original semiring (this arises from
|
||||
/// what happens when we combine identical arcs).
|
||||
/// (*) by "where possible".. there are situations where we wouldn't be able to
|
||||
/// preserve stochasticity in the LogArc sense while maintaining equivalence in
|
||||
/// the StdArc sense, so in these situations we maintain equivalence.
|
||||
|
||||
template<class Arc>
|
||||
void RemoveEpsLocal(MutableFst<Arc> *fst);
|
||||
|
||||
/// As RemoveEpsLocal but takes care to preserve stochasticity
|
||||
/// when cast to LogArc.
|
||||
inline void RemoveEpsLocalSpecial(MutableFst<StdArc> *fst);
|
||||
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#include "remove-eps-local-inl.h"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,257 @@
|
||||
// fstext/table-matcher-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fstext/table-matcher.h"
|
||||
#include "fstext/fst-test-utils.h"
|
||||
#include "base/kaldi-math.h"
|
||||
|
||||
namespace fst{
|
||||
|
||||
|
||||
// Don't instantiate with log semiring, as RandEquivalent may fail.
|
||||
template<class Arc> void TestTableMatcher(bool connect, bool left) {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
|
||||
VectorFst<Arc> *fst1 = RandFst<Arc>();
|
||||
|
||||
VectorFst<Arc> *fst2 = RandFst<Arc>();
|
||||
|
||||
ILabelCompare<Arc> ilabel_comp;
|
||||
OLabelCompare<Arc> olabel_comp;
|
||||
|
||||
TableComposeOptions opts;
|
||||
if (left) opts.table_match_type = MATCH_OUTPUT;
|
||||
else opts.table_match_type = MATCH_INPUT;
|
||||
opts.min_table_size = 1 + kaldi::Rand() % 5;
|
||||
opts.table_ratio = 0.25 * (kaldi::Rand() % 5);
|
||||
opts.connect = connect;
|
||||
|
||||
ArcSort(fst1, olabel_comp);
|
||||
ArcSort(fst2, ilabel_comp);
|
||||
|
||||
VectorFst<Arc> composed;
|
||||
|
||||
TableCompose(*fst1, *fst2, &composed, opts);
|
||||
|
||||
if (!connect) Connect(&composed);
|
||||
|
||||
VectorFst<Arc> composed_baseline;
|
||||
|
||||
Compose(*fst1, *fst2, &composed_baseline);
|
||||
|
||||
|
||||
std::cout << "Connect = "<< (connect?"True\n":"False\n");
|
||||
|
||||
std::cout <<"Table-Composed FST\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(composed, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
std::cout <<" Baseline-Composed FST\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(composed_baseline, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
if ( !RandEquivalent(composed, composed_baseline, 3/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 20/*path length-- max?*/)) {
|
||||
VectorFst<Arc> diff1;
|
||||
Difference(composed, composed_baseline, &diff1);
|
||||
std::cout <<" Diff1 (composed - baseline) \n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(diff1, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
VectorFst<Arc> diff2;
|
||||
Difference(composed_baseline, composed, &diff2);
|
||||
std::cout <<" Diff2 (baseline - composed) \n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(diff2, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
assert(0);
|
||||
}
|
||||
|
||||
delete fst1;
|
||||
delete fst2;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Don't instantiate with log semiring, as RandEquivalent may fail.
|
||||
template<class Arc> void TestTableMatcherCacheLeft(bool connect) {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
|
||||
VectorFst<Arc> *fst1 = RandFst<Arc>();
|
||||
|
||||
|
||||
TableComposeOptions opts;
|
||||
opts.table_match_type = MATCH_OUTPUT;
|
||||
opts.min_table_size = 1 + kaldi::Rand() % 5;
|
||||
opts.table_ratio = 0.25 * (kaldi::Rand() % 5);
|
||||
opts.connect = connect;
|
||||
|
||||
TableComposeCache<Fst<Arc> > cache(opts);
|
||||
|
||||
for (size_t i = 0; i < 3; i++) {
|
||||
|
||||
VectorFst<Arc> *fst2 = RandFst<Arc>();
|
||||
|
||||
ILabelCompare<Arc> ilabel_comp;
|
||||
OLabelCompare<Arc> olabel_comp;
|
||||
|
||||
|
||||
ArcSort(fst1, olabel_comp);
|
||||
ArcSort(fst2, ilabel_comp);
|
||||
|
||||
VectorFst<Arc> composed;
|
||||
|
||||
TableCompose(*fst1, *fst2, &composed, &cache);
|
||||
|
||||
if (!connect) Connect(&composed);
|
||||
|
||||
VectorFst<Arc> composed_baseline;
|
||||
|
||||
Compose(*fst1, *fst2, &composed_baseline);
|
||||
|
||||
|
||||
std::cout << "Connect = "<< (connect?"True\n":"False\n");
|
||||
|
||||
|
||||
if ( !RandEquivalent(composed, composed_baseline, 3/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/)) {
|
||||
VectorFst<Arc> diff1;
|
||||
Difference(composed, composed_baseline, &diff1);
|
||||
std::cout <<" Diff1 (composed - baseline) \n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(diff1, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
VectorFst<Arc> diff2;
|
||||
Difference(composed_baseline, composed, &diff2);
|
||||
std::cout <<" Diff2 (baseline - composed) \n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(diff2, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
assert(0);
|
||||
}
|
||||
delete fst2;
|
||||
}
|
||||
|
||||
delete fst1;
|
||||
}
|
||||
|
||||
|
||||
template<class Arc> void TestTableMatcherCacheRight(bool connect) {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
|
||||
VectorFst<Arc> *fst2 = RandFst<Arc>();
|
||||
ILabelCompare<Arc> ilabel_comp;
|
||||
ArcSort(fst2, ilabel_comp);
|
||||
|
||||
|
||||
TableComposeOptions opts;
|
||||
opts.table_match_type = MATCH_INPUT;
|
||||
opts.min_table_size = 1 + kaldi::Rand() % 5;
|
||||
opts.table_ratio = 0.25 * (kaldi::Rand() % 5);
|
||||
opts.connect = connect;
|
||||
|
||||
TableComposeCache<Fst<Arc> > cache(opts);
|
||||
|
||||
for (size_t i = 0; i < 2; i++) {
|
||||
|
||||
VectorFst<Arc> *fst1 = RandFst<Arc>();
|
||||
|
||||
|
||||
OLabelCompare<Arc> olabel_comp;
|
||||
|
||||
|
||||
ArcSort(fst1, olabel_comp);
|
||||
|
||||
VectorFst<Arc> composed;
|
||||
|
||||
TableCompose(*fst1, *fst2, &composed, &cache);
|
||||
|
||||
if (!connect) Connect(&composed);
|
||||
|
||||
VectorFst<Arc> composed_baseline;
|
||||
|
||||
Compose(*fst1, *fst2, &composed_baseline);
|
||||
|
||||
|
||||
std::cout << "Connect = "<< (connect?"True\n":"False\n");
|
||||
|
||||
|
||||
if ( !RandEquivalent(composed, composed_baseline, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 20/*path length-- max?*/)) {
|
||||
VectorFst<Arc> diff1;
|
||||
Difference(composed, composed_baseline, &diff1);
|
||||
std::cout <<" Diff1 (composed - baseline) \n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(diff1, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
VectorFst<Arc> diff2;
|
||||
Difference(composed_baseline, composed, &diff2);
|
||||
std::cout <<" Diff2 (baseline - composed) \n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(diff2, NULL, NULL, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
assert(0);
|
||||
}
|
||||
delete fst1;
|
||||
}
|
||||
|
||||
delete fst2;
|
||||
}
|
||||
|
||||
|
||||
} // namespace fst
|
||||
|
||||
int main() {
|
||||
using namespace fst;
|
||||
for (int i = 0;i < 1;i++) {
|
||||
TestTableMatcher<fst::StdArc>(true, true);
|
||||
TestTableMatcher<fst::StdArc>(false, true);
|
||||
TestTableMatcher<fst::StdArc>(true, false);
|
||||
TestTableMatcher<fst::StdArc>(false, false);
|
||||
TestTableMatcherCacheLeft<fst::StdArc>(true);
|
||||
TestTableMatcherCacheLeft<fst::StdArc>(false);
|
||||
TestTableMatcherCacheRight<fst::StdArc>(true);
|
||||
TestTableMatcherCacheRight<fst::StdArc>(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
// fstext/table-matcher.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_FSTEXT_TABLE_MATCHER_H_
|
||||
#define KALDI_FSTEXT_TABLE_MATCHER_H_
|
||||
#include <fst/fstlib.h>
|
||||
#include <fst/fst-decl.h>
|
||||
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
/// TableMatcher is a matcher specialized for the case where the output
|
||||
/// side of the left FST always has either all-epsilons coming out of
|
||||
/// a state, or a majority of the symbol table. Therefore we can
|
||||
/// either store nothing (for the all-epsilon case) or store a lookup
|
||||
/// table from Labels to arc offsets. Since the TableMatcher has to
|
||||
/// iterate over all arcs in each left-hand state the first time it sees
|
||||
/// it, this matcher type is not efficient if you compose with
|
||||
/// something very small on the right-- unless you do it multiple
|
||||
/// times and keep the matcher around. To do this requires using the
|
||||
/// most advanced form of ComposeFst in Compose.h, that initializes
|
||||
/// with ComposeFstImplOptions.
|
||||
|
||||
struct TableMatcherOptions {
|
||||
float table_ratio; // we construct the table if it would be at least this full.
|
||||
int min_table_size;
|
||||
TableMatcherOptions(): table_ratio(0.25), min_table_size(4) { }
|
||||
};
|
||||
|
||||
|
||||
// Introducing an "impl" class for TableMatcher because
|
||||
// we need to do a shallow copy of the Matcher for when
|
||||
// we want to cache tables for multiple compositions.
|
||||
template<class F, class BackoffMatcher = SortedMatcher<F> >
|
||||
class TableMatcherImpl : public MatcherBase<typename F::Arc> {
|
||||
public:
|
||||
typedef F FST;
|
||||
typedef typename F::Arc Arc;
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef StateId ArcId; // Use this type to store arc offsets [it's actually size_t
|
||||
// in the Seek function of ArcIterator, but StateId should be big enough].
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
|
||||
public:
|
||||
|
||||
TableMatcherImpl(const FST &fst, MatchType match_type,
|
||||
const TableMatcherOptions &opts = TableMatcherOptions()):
|
||||
match_type_(match_type),
|
||||
fst_(fst.Copy()),
|
||||
loop_(match_type == MATCH_INPUT ?
|
||||
Arc(kNoLabel, 0, Weight::One(), kNoStateId) :
|
||||
Arc(0, kNoLabel, Weight::One(), kNoStateId)),
|
||||
aiter_(NULL),
|
||||
s_(kNoStateId), opts_(opts),
|
||||
backoff_matcher_(fst, match_type)
|
||||
{
|
||||
assert(opts_.min_table_size > 0);
|
||||
if (match_type == MATCH_INPUT)
|
||||
assert(fst_->Properties(kILabelSorted, true) == kILabelSorted);
|
||||
else if (match_type == MATCH_OUTPUT)
|
||||
assert(fst_->Properties(kOLabelSorted, true) == kOLabelSorted);
|
||||
else
|
||||
assert(0 && "Invalid FST properties");
|
||||
}
|
||||
|
||||
virtual const FST &GetFst() const { return *fst_; }
|
||||
|
||||
virtual ~TableMatcherImpl() {
|
||||
std::vector<ArcId> *const empty = ((std::vector<ArcId>*)(NULL)) + 1; // special marker.
|
||||
for (size_t i = 0; i < tables_.size(); i++) {
|
||||
if (tables_[i] != NULL && tables_[i] != empty)
|
||||
delete tables_[i];
|
||||
}
|
||||
delete aiter_;
|
||||
delete fst_;
|
||||
}
|
||||
|
||||
virtual MatchType Type(bool test) const {
|
||||
return match_type_;
|
||||
}
|
||||
|
||||
void SetState(StateId s) {
|
||||
if (aiter_) {
|
||||
delete aiter_;
|
||||
aiter_ = NULL;
|
||||
}
|
||||
if (match_type_ == MATCH_NONE)
|
||||
LOG(FATAL) << "TableMatcher: bad match type";
|
||||
s_ = s;
|
||||
std::vector<ArcId> *const empty = ((std::vector<ArcId>*)(NULL)) + 1; // special marker.
|
||||
if (static_cast<size_t>(s) >= tables_.size()) {
|
||||
assert(s>=0);
|
||||
tables_.resize(s+1, NULL);
|
||||
}
|
||||
std::vector<ArcId>* &this_table_ = tables_[s]; // note: ref to ptr.
|
||||
if (this_table_ == empty) {
|
||||
backoff_matcher_.SetState(s);
|
||||
return;
|
||||
} else if (this_table_ == NULL) { // NULL means has not been set.
|
||||
ArcId num_arcs = fst_->NumArcs(s);
|
||||
if (num_arcs == 0 || num_arcs < opts_.min_table_size) {
|
||||
this_table_ = empty;
|
||||
backoff_matcher_.SetState(s);
|
||||
return;
|
||||
}
|
||||
ArcIterator<FST> aiter(*fst_, s);
|
||||
aiter.SetFlags(kArcNoCache|(match_type_ == MATCH_OUTPUT?kArcOLabelValue:kArcILabelValue),
|
||||
kArcNoCache|kArcValueFlags);
|
||||
// the statement above, says: "Don't cache stuff; and I only need the ilabel/olabel
|
||||
// to be computed.
|
||||
aiter.Seek(num_arcs - 1);
|
||||
Label highest_label = (match_type_ == MATCH_OUTPUT ?
|
||||
aiter.Value().olabel : aiter.Value().ilabel);
|
||||
if ((highest_label+1) * opts_.table_ratio > num_arcs) {
|
||||
this_table_ = empty;
|
||||
backoff_matcher_.SetState(s);
|
||||
return; // table would be too sparse.
|
||||
}
|
||||
// OK, now we are creating the table.
|
||||
this_table_ = new std::vector<ArcId> (highest_label+1, kNoStateId);
|
||||
ArcId pos = 0;
|
||||
for (aiter.Seek(0); !aiter.Done(); aiter.Next(), pos++) {
|
||||
Label label = (match_type_ == MATCH_OUTPUT ?
|
||||
aiter.Value().olabel : aiter.Value().ilabel);
|
||||
assert((size_t)label <= (size_t)highest_label); // also checks >= 0.
|
||||
if ((*this_table_)[label] == kNoStateId) (*this_table_)[label] = pos;
|
||||
// set this_table_[label] to first position where arc has this
|
||||
// label.
|
||||
}
|
||||
}
|
||||
// At this point in the code, this_table_ != NULL and != empty.
|
||||
aiter_ = new ArcIterator<FST>(*fst_, s);
|
||||
aiter_->SetFlags(kArcNoCache, kArcNoCache); // don't need to cache arcs as may only
|
||||
// need a small subset.
|
||||
loop_.nextstate = s;
|
||||
// aiter_ = NULL;
|
||||
// backoff_matcher_.SetState(s);
|
||||
}
|
||||
|
||||
bool Find(Label match_label) {
|
||||
if (!aiter_) return backoff_matcher_.Find(match_label);
|
||||
else {
|
||||
match_label_ = match_label;
|
||||
current_loop_ = (match_label == 0);
|
||||
// kNoLabel means the implicit loop on the other FST --
|
||||
// matches real epsilons but not the self-loop.
|
||||
match_label_ = (match_label_ == kNoLabel ? 0 : match_label_);
|
||||
if (static_cast<size_t>(match_label_) < tables_[s_]->size() &&
|
||||
(*(tables_[s_]))[match_label_] != kNoStateId) {
|
||||
aiter_->Seek( (*(tables_[s_]))[match_label_] ); // label exists.
|
||||
return true;
|
||||
}
|
||||
return current_loop_;
|
||||
}
|
||||
}
|
||||
const Arc& Value() const {
|
||||
if (aiter_)
|
||||
return current_loop_ ? loop_ : aiter_->Value();
|
||||
else
|
||||
return backoff_matcher_.Value();
|
||||
}
|
||||
|
||||
void Next() {
|
||||
if (aiter_) {
|
||||
if (current_loop_)
|
||||
current_loop_ = false;
|
||||
else
|
||||
aiter_->Next();
|
||||
} else
|
||||
backoff_matcher_.Next();
|
||||
}
|
||||
|
||||
bool Done() const {
|
||||
if (aiter_ != NULL) {
|
||||
if (current_loop_)
|
||||
return false;
|
||||
if (aiter_->Done())
|
||||
return true;
|
||||
Label label = (match_type_ == MATCH_OUTPUT ?
|
||||
aiter_->Value().olabel : aiter_->Value().ilabel);
|
||||
return (label != match_label_);
|
||||
} else
|
||||
return backoff_matcher_.Done();
|
||||
}
|
||||
const Arc &Value() {
|
||||
if (aiter_ != NULL) {
|
||||
return (current_loop_ ? loop_ : aiter_->Value() );
|
||||
} else
|
||||
return backoff_matcher_.Value();
|
||||
}
|
||||
|
||||
virtual TableMatcherImpl<FST> *Copy(bool safe = false) const {
|
||||
assert(0); // shouldn't be called. This is not a "real" matcher,
|
||||
// although we derive from MatcherBase for convenience.
|
||||
return NULL;
|
||||
}
|
||||
|
||||
virtual uint64 Properties(uint64 props) const { return props; } // simple matcher that does
|
||||
// not change its FST, so properties are properties of FST it is applied to
|
||||
|
||||
private:
|
||||
virtual void SetState_(StateId s) { SetState(s); }
|
||||
virtual bool Find_(Label label) { return Find(label); }
|
||||
virtual bool Done_() const { return Done(); }
|
||||
virtual const Arc& Value_() const { return Value(); }
|
||||
virtual void Next_() { Next(); }
|
||||
|
||||
MatchType match_type_;
|
||||
FST *fst_;
|
||||
bool current_loop_;
|
||||
Label match_label_;
|
||||
Arc loop_;
|
||||
ArcIterator<FST> *aiter_;
|
||||
StateId s_;
|
||||
std::vector<std::vector<ArcId> *> tables_;
|
||||
TableMatcherOptions opts_;
|
||||
BackoffMatcher backoff_matcher_;
|
||||
|
||||
};
|
||||
|
||||
|
||||
template<class F, class BackoffMatcher = SortedMatcher<F> >
|
||||
class TableMatcher : public MatcherBase<typename F::Arc> {
|
||||
public:
|
||||
typedef F FST;
|
||||
typedef typename F::Arc Arc;
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef StateId ArcId; // Use this type to store arc offsets [it's actually size_t
|
||||
// in the Seek function of ArcIterator, but StateId should be big enough].
|
||||
typedef typename Arc::Weight Weight;
|
||||
typedef TableMatcherImpl<F, BackoffMatcher> Impl;
|
||||
|
||||
TableMatcher(const FST &fst, MatchType match_type,
|
||||
const TableMatcherOptions &opts = TableMatcherOptions())
|
||||
: impl_(std::make_shared<Impl>(fst, match_type, opts)) { }
|
||||
|
||||
TableMatcher(const TableMatcher<FST, BackoffMatcher> &matcher,
|
||||
bool safe = false)
|
||||
: impl_(matcher.impl_) {
|
||||
if (safe == true) {
|
||||
LOG(FATAL) << "TableMatcher: Safe copy not supported";
|
||||
}
|
||||
}
|
||||
|
||||
virtual const FST &GetFst() const { return impl_->GetFst(); }
|
||||
|
||||
virtual MatchType Type(bool test) const { return impl_->Type(test); }
|
||||
|
||||
void SetState(StateId s) { return impl_->SetState(s); }
|
||||
|
||||
bool Find(Label match_label) { return impl_->Find(match_label); }
|
||||
|
||||
const Arc& Value() const { return impl_->Value(); }
|
||||
|
||||
void Next() { return impl_->Next(); }
|
||||
|
||||
bool Done() const { return impl_->Done(); }
|
||||
|
||||
const Arc &Value() { return impl_->Value(); }
|
||||
|
||||
virtual TableMatcher<FST, BackoffMatcher> *Copy(bool safe = false) const {
|
||||
return new TableMatcher<FST, BackoffMatcher> (*this, safe);
|
||||
}
|
||||
|
||||
virtual uint64 Properties(uint64 props) const { return impl_->Properties(props); } // simple matcher that does
|
||||
// not change its FST, so properties are properties of FST it is applied to
|
||||
private:
|
||||
std::shared_ptr<Impl> impl_;
|
||||
|
||||
virtual void SetState_(StateId s) { impl_->SetState(s); }
|
||||
virtual bool Find_(Label label) { return impl_->Find(label); }
|
||||
virtual bool Done_() const { return impl_->Done(); }
|
||||
virtual const Arc& Value_() const { return impl_->Value(); }
|
||||
virtual void Next_() { impl_->Next(); }
|
||||
|
||||
TableMatcher &operator=(const TableMatcher &) = delete;
|
||||
};
|
||||
|
||||
struct TableComposeOptions: public TableMatcherOptions {
|
||||
bool connect; // Connect output
|
||||
ComposeFilter filter_type; // Which pre-defined filter to use
|
||||
MatchType table_match_type;
|
||||
|
||||
explicit TableComposeOptions(const TableMatcherOptions &mo,
|
||||
bool c = true, ComposeFilter ft = SEQUENCE_FILTER,
|
||||
MatchType tms = MATCH_OUTPUT)
|
||||
: TableMatcherOptions(mo), connect(c), filter_type(ft), table_match_type(tms) { }
|
||||
TableComposeOptions() : connect(true), filter_type(SEQUENCE_FILTER),
|
||||
table_match_type(MATCH_OUTPUT) { }
|
||||
};
|
||||
|
||||
|
||||
template<class Arc>
|
||||
void TableCompose(const Fst<Arc> &ifst1, const Fst<Arc> &ifst2,
|
||||
MutableFst<Arc> *ofst,
|
||||
const TableComposeOptions &opts = TableComposeOptions()) {
|
||||
typedef Fst<Arc> F;
|
||||
CacheOptions nopts;
|
||||
nopts.gc_limit = 0; // Cache only the last state for fastest copy.
|
||||
if (opts.table_match_type == MATCH_OUTPUT) {
|
||||
// ComposeFstImplOptions templated on matcher for fst1, matcher for fst2.
|
||||
ComposeFstImplOptions<TableMatcher<F>, SortedMatcher<F> > impl_opts(nopts);
|
||||
impl_opts.matcher1 = new TableMatcher<F>(ifst1, MATCH_OUTPUT, opts);
|
||||
*ofst = ComposeFst<Arc>(ifst1, ifst2, impl_opts);
|
||||
} else {
|
||||
assert(opts.table_match_type == MATCH_INPUT) ;
|
||||
// ComposeFstImplOptions templated on matcher for fst1, matcher for fst2.
|
||||
ComposeFstImplOptions<SortedMatcher<F>, TableMatcher<F> > impl_opts(nopts);
|
||||
impl_opts.matcher2 = new TableMatcher<F>(ifst2, MATCH_INPUT, opts);
|
||||
*ofst = ComposeFst<Arc>(ifst1, ifst2, impl_opts);
|
||||
}
|
||||
if (opts.connect) Connect(ofst);
|
||||
}
|
||||
|
||||
|
||||
/// TableComposeCache lets us do multiple compositions while caching the same
|
||||
/// matcher.
|
||||
template<class F>
|
||||
struct TableComposeCache {
|
||||
TableMatcher<F> *matcher;
|
||||
TableComposeOptions opts;
|
||||
TableComposeCache(const TableComposeOptions &opts = TableComposeOptions()): matcher (NULL), opts(opts) {}
|
||||
~TableComposeCache() { delete(matcher); }
|
||||
};
|
||||
|
||||
template<class Arc>
|
||||
void TableCompose(const Fst<Arc> &ifst1, const Fst<Arc> &ifst2,
|
||||
MutableFst<Arc> *ofst,
|
||||
TableComposeCache<Fst<Arc> > *cache) {
|
||||
typedef Fst<Arc> F;
|
||||
assert(cache != NULL);
|
||||
CacheOptions nopts;
|
||||
nopts.gc_limit = 0; // Cache only the last state for fastest copy.
|
||||
if (cache->opts.table_match_type == MATCH_OUTPUT) {
|
||||
ComposeFstImplOptions<TableMatcher<F>, SortedMatcher<F> > impl_opts(nopts);
|
||||
if (cache->matcher == NULL)
|
||||
cache->matcher = new TableMatcher<F>(ifst1, MATCH_OUTPUT, cache->opts);
|
||||
impl_opts.matcher1 = cache->matcher->Copy(); // not passing "safe": may not
|
||||
// be thread-safe-- anway I don't understand this part.
|
||||
*ofst = ComposeFst<Arc>(ifst1, ifst2, impl_opts);
|
||||
} else {
|
||||
assert(cache->opts.table_match_type == MATCH_INPUT) ;
|
||||
ComposeFstImplOptions<SortedMatcher<F>, TableMatcher<F> > impl_opts(nopts);
|
||||
if (cache->matcher == NULL)
|
||||
cache->matcher = new TableMatcher<F>(ifst2, MATCH_INPUT, cache->opts);
|
||||
impl_opts.matcher2 = cache->matcher->Copy();
|
||||
*ofst = ComposeFst<Arc>(ifst1, ifst2, impl_opts);
|
||||
}
|
||||
if (cache->opts.connect) Connect(ofst);
|
||||
}
|
||||
|
||||
|
||||
|
||||
} // end namespace fst
|
||||
#endif
|
||||
@@ -0,0 +1,214 @@
|
||||
// fstext/trivial-factor-weight-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "base/kaldi-math.h"
|
||||
#include "fstext/pre-determinize.h"
|
||||
#include "fstext/determinize-star.h"
|
||||
#include "fstext/trivial-factor-weight.h"
|
||||
#include "fstext/fst-test-utils.h"
|
||||
// Just check that it compiles, for now.
|
||||
|
||||
namespace fst
|
||||
{
|
||||
using std::cout;
|
||||
using std::vector;
|
||||
|
||||
// Don't instantiate with log semiring, as RandEquivalent may fail.
|
||||
template<class Arc> void TestFactor() {
|
||||
typedef typename Arc::Label Label;
|
||||
typedef typename Arc::StateId StateId;
|
||||
typedef typename Arc::Weight Weight;
|
||||
|
||||
VectorFst<Arc> *fst = new VectorFst<Arc>();
|
||||
int n_syms = 2 + kaldi::Rand() % 5, n_states = 3 + kaldi::Rand() % 10, n_arcs = 5 + kaldi::Rand() % 30, n_final = 1 + kaldi::Rand()%3; // Up to 2 unique symbols.
|
||||
cout << "Testing pre-determinize with "<<n_syms<<" symbols, "<<n_states<<" states and "<<n_arcs<<" arcs and "<<n_final<<" final states.\n";
|
||||
SymbolTable *sptr = NULL;
|
||||
|
||||
vector<Label> all_syms; // including epsilon.
|
||||
// Put symbols in the symbol table from 1..n_syms-1.
|
||||
for (size_t i = 0;i < (size_t)n_syms;i++)
|
||||
all_syms.push_back(i);
|
||||
|
||||
// Create states.
|
||||
vector<StateId> all_states;
|
||||
for (size_t i = 0;i < (size_t)n_states;i++) {
|
||||
StateId this_state = fst->AddState();
|
||||
if (i == 0) fst->SetStart(i);
|
||||
all_states.push_back(this_state);
|
||||
}
|
||||
// Set final states.
|
||||
for (size_t j = 0;j < (size_t)n_final;j++) {
|
||||
StateId id = all_states[kaldi::Rand() % n_states];
|
||||
Weight weight = (Weight)(0.33*(kaldi::Rand() % 5) );
|
||||
printf("calling SetFinal with %d and %f\n", id, weight.Value());
|
||||
fst->SetFinal(id, weight);
|
||||
}
|
||||
// Create arcs.
|
||||
for (size_t i = 0;i < (size_t)n_arcs;i++) {
|
||||
Arc a;
|
||||
a.nextstate = all_states[kaldi::Rand() % n_states];
|
||||
a.ilabel = all_syms[kaldi::Rand() % n_syms];
|
||||
a.olabel = all_syms[kaldi::Rand() % n_syms]; // same input+output vocab.
|
||||
a.weight = (Weight) (0.33*(kaldi::Rand() % 2));
|
||||
StateId start_state = all_states[kaldi::Rand() % n_states];
|
||||
fst->AddArc(start_state, a);
|
||||
}
|
||||
|
||||
std::cout <<" printing before trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
// Trim resulting FST.
|
||||
Connect(fst);
|
||||
|
||||
std::cout <<" printing after trimming\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
vector<Label> extra_syms;
|
||||
if (fst->Start() != kNoStateId) { // "Connect" did not make it empty....
|
||||
PreDeterminize(fst, 1000, &extra_syms);
|
||||
}
|
||||
|
||||
std::cout <<" printing after predeterminization\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
{ // Remove epsilon. All default args.
|
||||
bool connect = true;
|
||||
Weight weight_threshold = Weight::Zero();
|
||||
int64 nstate = -1; // Relates to pruning.
|
||||
double delta = kDelta; // I think a small weight value. Relates to some kind of pruning,
|
||||
// I guess. But with no epsilon cycles, probably doensn't matter.
|
||||
RmEpsilon(fst, connect, weight_threshold, nstate, delta);
|
||||
}
|
||||
|
||||
std::cout <<" printing after double-epsilon removal\n";
|
||||
{
|
||||
FstPrinter<Arc> fstprinter(*fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
VectorFst<Arc> ofst_star;
|
||||
|
||||
{
|
||||
printf("Converting to Gallic semiring");
|
||||
VectorFst<GallicArc<Arc> > gallic_fst;
|
||||
VectorFst<GallicArc<Arc> > gallic_fst_noeps;
|
||||
VectorFst<GallicArc<Arc> > gallic_fst_det;
|
||||
|
||||
|
||||
{
|
||||
printf("Determinizing with DeterminizeStar, converting to Gallic\n");
|
||||
DeterminizeStar(*fst, &gallic_fst);
|
||||
}
|
||||
|
||||
{
|
||||
std::cout <<" printing gallic FST\n";
|
||||
FstPrinter<GallicArc<Arc> > fstprinter(gallic_fst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
// Map(ofst_star, &gallic_fst, ToGallicMapper<Arc, STRING_LEFT>());
|
||||
|
||||
printf("Converting gallic back to regular\n");
|
||||
TrivialFactorWeightFst< GallicArc<Arc, GALLIC_LEFT>, GallicFactor<typename Arc::Label,
|
||||
typename Arc::Weight, GALLIC_LEFT> > fwfst(gallic_fst);
|
||||
{
|
||||
std::cout <<" printing factor-weight FST\n";
|
||||
FstPrinter<GallicArc<Arc> > fstprinter(fwfst, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
Map(fwfst, &ofst_star, FromGallicMapper<Arc, GALLIC_LEFT>());
|
||||
|
||||
{
|
||||
std::cout <<" printing after converting back to regular FST\n";
|
||||
FstPrinter<Arc> fstprinter(ofst_star, sptr, sptr, NULL, false, true, "\t");
|
||||
fstprinter.Print(&std::cout, "standard output");
|
||||
}
|
||||
|
||||
|
||||
VectorFst<GallicArc<Arc> > new_gallic_fst;
|
||||
Map(ofst_star, &new_gallic_fst, ToGallicMapper<Arc, GALLIC_LEFT>());
|
||||
|
||||
assert(RandEquivalent(gallic_fst, new_gallic_fst, 5/*paths*/, 0.01/*delta*/, kaldi::Rand()/*seed*/, 100/*path length-- max?*/));
|
||||
|
||||
}
|
||||
|
||||
delete fst;
|
||||
}
|
||||
|
||||
|
||||
template<class Arc, class inttype> void TestStringRepository() {
|
||||
typedef typename Arc::Label Label;
|
||||
|
||||
StringRepository<Label, inttype> sr;
|
||||
|
||||
int N = 1000;
|
||||
if (sizeof(inttype) == 1) N = 64;
|
||||
vector<vector<Label> > strings(N);
|
||||
vector<inttype> ids(N);
|
||||
|
||||
for (size_t i = 0;i < N;i++) {
|
||||
size_t len = kaldi::Rand() % 4;
|
||||
vector<Label> vec;
|
||||
for (size_t j = 0;j < len;j++) vec.push_back( (kaldi::Rand()%10) + 150*(kaldi::Rand()%2)); // make it have reasonable range.
|
||||
if (i < 500 && vec.size() == 0) ids[i] = sr.IdOfEmpty();
|
||||
else if (i < 500 && vec.size() == 1) ids[i] = sr.IdOfLabel(vec[0]);
|
||||
else ids[i] = sr.IdOfSeq(vec);
|
||||
|
||||
strings[i] = vec;
|
||||
}
|
||||
|
||||
for (size_t i = 0;i < N;i++) {
|
||||
vector<Label> tmpv;
|
||||
tmpv.push_back(10); // just put in garbage.
|
||||
sr.SeqOfId(ids[i], &tmpv);
|
||||
assert(tmpv == strings[i]);
|
||||
assert(sr.IdOfSeq(strings[i]) == ids[i]);
|
||||
if (strings[i].size() == 0) assert(ids[i] == sr.IdOfEmpty());
|
||||
if (strings[i].size() == 1) assert(ids[i] == sr.IdOfLabel(strings[i][0]));
|
||||
|
||||
if (sizeof(inttype) != 1) {
|
||||
size_t prefix_len = kaldi::Rand() % (strings[i].size() + 1);
|
||||
inttype s2 = sr.RemovePrefix(ids[i], prefix_len);
|
||||
vector<Label> vec2;
|
||||
sr.SeqOfId(s2, &vec2);
|
||||
for (size_t j = 0;j < strings[i].size()-prefix_len;j++) {
|
||||
assert(vec2[j] == strings[i][j+prefix_len]);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
} // end namespace fst
|
||||
|
||||
int main() {
|
||||
for (int i = 0;i < 25;i++) {
|
||||
fst::TestFactor<fst::StdArc>();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
// fstext/trivial-factor-weight.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
//
|
||||
// This is a modified file from the OpenFST Library v1.2.7 available at
|
||||
// http://www.openfst.org and released under the Apache License Version 2.0.
|
||||
//
|
||||
//
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Copyright 2005-2010 Google, Inc.
|
||||
// Author: allauzen@google.com (Cyril Allauzen)
|
||||
|
||||
|
||||
#ifndef KALDI_FSTEXT_TRIVIAL_FACTOR_WEIGHT_H_
|
||||
#define KALDI_FSTEXT_TRIVIAL_FACTOR_WEIGHT_H_
|
||||
|
||||
|
||||
// TrivialFactorWeight.h This is an extension to factor-weight.h in the OpenFst
|
||||
// code. It is a version of FactorWeight that creates separate states (with
|
||||
// input epsilons) rather than pushing the factors forward. This is for
|
||||
// converting from Gallic FSTs, where you want the result to be a bit more
|
||||
// trivial with input epsilons inserted where there are multiple output symbols.
|
||||
// This has the advantage that it always works, for any input (also I just
|
||||
// prefer this approach).
|
||||
|
||||
#include <unordered_map>
|
||||
using std::unordered_map;
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <fst/cache.h>
|
||||
#include <fst/test-properties.h>
|
||||
|
||||
namespace fst {
|
||||
|
||||
|
||||
template <class Arc>
|
||||
struct TrivialFactorWeightOptions : CacheOptions {
|
||||
typedef typename Arc::Label Label;
|
||||
float delta;
|
||||
Label extra_ilabel; // input label of extra arcs
|
||||
Label extra_olabel; // output label of extra arcs
|
||||
|
||||
TrivialFactorWeightOptions(const CacheOptions &opts, float d,
|
||||
Label il = 0, Label ol = 0)
|
||||
: CacheOptions(opts), delta(d), extra_ilabel(il), extra_olabel(ol) {}
|
||||
|
||||
explicit TrivialFactorWeightOptions(
|
||||
float d, Label il = 0, Label ol = 0)
|
||||
: delta(d), extra_ilabel(il), extra_olabel(ol) {}
|
||||
|
||||
TrivialFactorWeightOptions(): delta(kDelta), extra_ilabel(0), extra_olabel(0) {}
|
||||
|
||||
};
|
||||
|
||||
namespace internal {
|
||||
|
||||
// Implementation class for TrivialFactorWeight
|
||||
template <class A, class F>
|
||||
class TrivialFactorWeightFstImpl
|
||||
: public CacheImpl<A> {
|
||||
public:
|
||||
using CacheImpl<A>::PushArc;
|
||||
using FstImpl<A>::SetType;
|
||||
using FstImpl<A>::SetProperties;
|
||||
using FstImpl<A>::Properties;
|
||||
using FstImpl<A>::SetInputSymbols;
|
||||
using FstImpl<A>::SetOutputSymbols;
|
||||
|
||||
using CacheBaseImpl< CacheState<A> >::HasStart;
|
||||
using CacheBaseImpl< CacheState<A> >::HasFinal;
|
||||
using CacheBaseImpl< CacheState<A> >::HasArcs;
|
||||
|
||||
typedef A Arc;
|
||||
typedef typename A::Label Label;
|
||||
typedef typename A::Weight Weight;
|
||||
typedef typename A::StateId StateId;
|
||||
typedef F FactorIterator;
|
||||
|
||||
typedef DefaultCacheStore<A> Store;
|
||||
typedef typename Store::State State;
|
||||
|
||||
struct Element {
|
||||
Element() {}
|
||||
|
||||
Element(StateId s, Weight w) : state(s), weight(w) {}
|
||||
|
||||
StateId state; // Input state Id
|
||||
Weight weight; // Residual weight
|
||||
};
|
||||
|
||||
TrivialFactorWeightFstImpl(const Fst<A> &fst, const TrivialFactorWeightOptions<A> &opts)
|
||||
: CacheImpl<A>(opts),
|
||||
fst_(fst.Copy()),
|
||||
delta_(opts.delta),
|
||||
extra_ilabel_(opts.extra_ilabel),
|
||||
extra_olabel_(opts.extra_olabel) {
|
||||
SetType("factor-weight");
|
||||
uint64 props = fst.Properties(kFstProperties, false);
|
||||
SetProperties(FactorWeightProperties(props), kCopyProperties);
|
||||
|
||||
SetInputSymbols(fst.InputSymbols());
|
||||
SetOutputSymbols(fst.OutputSymbols());
|
||||
}
|
||||
|
||||
TrivialFactorWeightFstImpl(const TrivialFactorWeightFstImpl<A, F> &impl)
|
||||
: CacheImpl<A>(impl),
|
||||
fst_(impl.fst_->Copy(true)),
|
||||
delta_(impl.delta_),
|
||||
extra_ilabel_(impl.extra_ilabel_),
|
||||
extra_olabel_(impl.extra_olabel_) {
|
||||
SetType("factor-weight");
|
||||
SetProperties(impl.Properties(), kCopyProperties);
|
||||
SetInputSymbols(impl.InputSymbols());
|
||||
SetOutputSymbols(impl.OutputSymbols());
|
||||
}
|
||||
|
||||
StateId Start() {
|
||||
if (!HasStart()) {
|
||||
StateId s = fst_->Start();
|
||||
if (s == kNoStateId)
|
||||
return kNoStateId;
|
||||
StateId start = this->FindState(Element(fst_->Start(), Weight::One()));
|
||||
this->SetStart(start);
|
||||
}
|
||||
return CacheImpl<A>::Start();
|
||||
}
|
||||
|
||||
Weight Final(StateId s) {
|
||||
if (!HasFinal(s)) {
|
||||
const Element &e = elements_[s];
|
||||
Weight w;
|
||||
if (e.state == kNoStateId) { // extra state inserted to represent final weights.
|
||||
FactorIterator fit(e.weight);
|
||||
if (fit.Done()) { // cannot be factored.
|
||||
w = e.weight; // so it's final
|
||||
} else {
|
||||
w = Weight::Zero(); // need another transition.
|
||||
}
|
||||
} else {
|
||||
if (e.weight != Weight::One()) { // Not a real state.
|
||||
w = Weight::Zero();
|
||||
} else { // corresponds to a "real" state.
|
||||
w = fst_->Final(e.state);
|
||||
FactorIterator fit(w);
|
||||
if (!fit.Done()) // we would have intermediate states representing this final state.
|
||||
w = Weight::Zero();
|
||||
}
|
||||
}
|
||||
this->SetFinal(s, w);
|
||||
return w;
|
||||
} else {
|
||||
return CacheImpl<A>::Final(s);
|
||||
}
|
||||
}
|
||||
|
||||
size_t NumArcs(StateId s) {
|
||||
if (!HasArcs(s))
|
||||
Expand(s);
|
||||
return CacheImpl<A>::NumArcs(s);
|
||||
}
|
||||
|
||||
size_t NumInputEpsilons(StateId s) {
|
||||
if (!HasArcs(s))
|
||||
Expand(s);
|
||||
return CacheImpl<A>::NumInputEpsilons(s);
|
||||
}
|
||||
|
||||
size_t NumOutputEpsilons(StateId s) {
|
||||
if (!HasArcs(s))
|
||||
Expand(s);
|
||||
return CacheImpl<A>::NumOutputEpsilons(s);
|
||||
}
|
||||
|
||||
void InitArcIterator(StateId s, ArcIteratorData<A> *data) {
|
||||
if (!HasArcs(s))
|
||||
Expand(s);
|
||||
CacheImpl<A>::InitArcIterator(s, data);
|
||||
}
|
||||
|
||||
|
||||
// Find state corresponding to an element. Create new state
|
||||
// if element not found.
|
||||
StateId FindState(const Element &e) {
|
||||
typename ElementMap::iterator eit = element_map_.find(e);
|
||||
if (eit != element_map_.end()) {
|
||||
return (*eit).second;
|
||||
} else {
|
||||
StateId s = elements_.size();
|
||||
elements_.push_back(e);
|
||||
element_map_.insert(std::pair<const Element, StateId>(e, s));
|
||||
return s;
|
||||
}
|
||||
}
|
||||
|
||||
// Computes the outgoing transitions from a state, creating new destination
|
||||
// states as needed.
|
||||
void Expand(StateId s) {
|
||||
CHECK(static_cast<size_t>(s) < elements_.size());
|
||||
Element e = elements_[s];
|
||||
if (e.weight != Weight::One()) {
|
||||
FactorIterator fit(e.weight);
|
||||
if (fit.Done()) { // Cannot be factored-> create a link to dest state directly
|
||||
if (e.state != kNoStateId) {
|
||||
StateId dest = FindState(Element(e.state, Weight::One()));
|
||||
PushArc(s, Arc(extra_ilabel_, extra_olabel_, e.weight, dest));
|
||||
} // else we're done. This is a final state.
|
||||
} else { // Can be factored.
|
||||
const std::pair<Weight, Weight> &p = fit.Value();
|
||||
StateId dest = FindState(Element(e.state, p.second.Quantize(delta_)));
|
||||
PushArc(s, Arc(extra_ilabel_, extra_olabel_, p.first, dest));
|
||||
}
|
||||
} else { // Unit weight. This corresponds to a "real" state.
|
||||
CHECK(e.state != kNoStateId);
|
||||
for (ArcIterator< Fst<A> > ait(*fst_, e.state);
|
||||
!ait.Done();
|
||||
ait.Next()) {
|
||||
const A &arc = ait.Value();
|
||||
FactorIterator fit(arc.weight);
|
||||
if (fit.Done()) { // cannot be factored->just link directly to dest.
|
||||
StateId dest = FindState(Element(arc.nextstate, Weight::One()));
|
||||
PushArc(s, Arc(arc.ilabel, arc.olabel, arc.weight, dest));
|
||||
} else {
|
||||
const std::pair<Weight, Weight> &p = fit.Value();
|
||||
StateId dest = FindState(Element(arc.nextstate, p.second.Quantize(delta_)));
|
||||
PushArc(s, Arc(arc.ilabel, arc.olabel, p.first, dest));
|
||||
}
|
||||
}
|
||||
// See if we have to add arcs for final-states [only if final-weight is factorable].
|
||||
Weight final_w = fst_->Final(e.state);
|
||||
if (final_w != Weight::Zero()) {
|
||||
FactorIterator fit(final_w);
|
||||
if (!fit.Done()) {
|
||||
const std::pair<Weight, Weight> &p = fit.Value();
|
||||
StateId dest = FindState(Element(kNoStateId, p.second.Quantize(delta_)));
|
||||
PushArc(s, Arc(extra_ilabel_, extra_olabel_, p.first, dest));
|
||||
}
|
||||
}
|
||||
}
|
||||
this->SetArcs(s);
|
||||
}
|
||||
|
||||
private:
|
||||
// Equality function for Elements, assume weights have been quantized.
|
||||
class ElementEqual {
|
||||
public:
|
||||
bool operator()(const Element &x, const Element &y) const {
|
||||
return x.state == y.state && x.weight == y.weight;
|
||||
}
|
||||
};
|
||||
|
||||
// Hash function for Elements to Fst states.
|
||||
class ElementKey {
|
||||
public:
|
||||
size_t operator()(const Element &x) const {
|
||||
return static_cast<size_t>(x.state * kPrime + x.weight.Hash());
|
||||
}
|
||||
private:
|
||||
static const int kPrime = 7853;
|
||||
};
|
||||
|
||||
typedef unordered_map<Element, StateId, ElementKey, ElementEqual> ElementMap;
|
||||
|
||||
std::unique_ptr<const Fst<A>> fst_;
|
||||
float delta_;
|
||||
uint32 mode_; // factoring arc and/or final weights
|
||||
Label extra_ilabel_; // ilabel of arc created when factoring final w's
|
||||
Label extra_olabel_; // olabel of arc created when factoring final w's
|
||||
std::vector<Element> elements_; // mapping Fst state to Elements
|
||||
ElementMap element_map_; // mapping Elements to Fst state
|
||||
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
|
||||
/// TrivialFactorWeightFst takes as template parameter a FactorIterator as
|
||||
/// defined above. The result of weight factoring is a transducer
|
||||
/// equivalent to the input whose path weights have been factored
|
||||
/// according to the FactorIterator. States and transitions will be
|
||||
/// added as necessary.
|
||||
/// This algorithm differs from the one implemented in FactorWeightFst
|
||||
/// in that it does not attempt to push the extra weight forward to the
|
||||
/// next state: it uses a sequence of "extra" intermediate state, and
|
||||
/// outputs the remaining weight right away. This ensures that it will
|
||||
/// always succeed, even for Gallic representations of FSTs that have cycles
|
||||
/// with more output than input symbols.
|
||||
|
||||
/// Note that the code below was modified from factor-weight.h by just
|
||||
/// search-and-replacing "FactorWeight" by "TrivialFactorWeight".
|
||||
|
||||
|
||||
template <class A, class F>
|
||||
class TrivialFactorWeightFst :
|
||||
public ImplToFst<internal::TrivialFactorWeightFstImpl<A, F>> {
|
||||
public:
|
||||
friend class ArcIterator< TrivialFactorWeightFst<A, F> >;
|
||||
friend class StateIterator< TrivialFactorWeightFst<A, F> >;
|
||||
|
||||
typedef A Arc;
|
||||
typedef typename A::Weight Weight;
|
||||
typedef typename A::StateId StateId;
|
||||
typedef DefaultCacheStore<Arc> Store;
|
||||
typedef typename Store::State State;
|
||||
typedef internal::TrivialFactorWeightFstImpl<A, F> Impl;
|
||||
|
||||
explicit TrivialFactorWeightFst(const Fst<A> &fst)
|
||||
: ImplToFst<Impl>(std::make_shared<Impl>(fst, TrivialFactorWeightOptions<A>())) {}
|
||||
|
||||
TrivialFactorWeightFst(const Fst<A> &fst, const TrivialFactorWeightOptions<A> &opts)
|
||||
: ImplToFst<Impl>(std::make_shared<Impl>(fst, opts)) {}
|
||||
|
||||
// See Fst<>::Copy() for doc.
|
||||
TrivialFactorWeightFst(const TrivialFactorWeightFst<A, F> &fst, bool copy)
|
||||
: ImplToFst<Impl>(fst, copy) {}
|
||||
|
||||
// Get a copy of this TrivialFactorWeightFst. See Fst<>::Copy() for further doc.
|
||||
TrivialFactorWeightFst<A, F> *Copy(bool copy = false) const override {
|
||||
return new TrivialFactorWeightFst<A, F>(*this, copy);
|
||||
}
|
||||
|
||||
inline void InitStateIterator(StateIteratorData<A> *data) const override;
|
||||
|
||||
void InitArcIterator(StateId s, ArcIteratorData<A> *data) const override {
|
||||
GetMutableImpl()->InitArcIterator(s, data);
|
||||
}
|
||||
|
||||
private:
|
||||
using ImplToFst<Impl>::GetImpl;
|
||||
using ImplToFst<Impl>::GetMutableImpl;
|
||||
|
||||
TrivialFactorWeightFst &operator=(const TrivialFactorWeightFst &fst) = delete;
|
||||
};
|
||||
|
||||
|
||||
// Specialization for TrivialFactorWeightFst.
|
||||
template<class A, class F>
|
||||
class StateIterator< TrivialFactorWeightFst<A, F> >
|
||||
: public CacheStateIterator< TrivialFactorWeightFst<A, F> > {
|
||||
public:
|
||||
explicit StateIterator(const TrivialFactorWeightFst<A, F> &fst)
|
||||
: CacheStateIterator< TrivialFactorWeightFst<A, F> >(fst, fst.GetMutableImpl()) {}
|
||||
};
|
||||
|
||||
|
||||
// Specialization for TrivialFactorWeightFst.
|
||||
template <class A, class F>
|
||||
class ArcIterator< TrivialFactorWeightFst<A, F> >
|
||||
: public CacheArcIterator< TrivialFactorWeightFst<A, F> > {
|
||||
public:
|
||||
typedef typename A::StateId StateId;
|
||||
|
||||
ArcIterator(const TrivialFactorWeightFst<A, F> &fst, StateId s)
|
||||
: CacheArcIterator< TrivialFactorWeightFst<A, F>>(fst.GetMutableImpl(), s) {
|
||||
if (!fst.GetImpl()->HasArcs(s)) fst.GetMutableImpl()->Expand(s);
|
||||
}
|
||||
};
|
||||
|
||||
template <class A, class F>
|
||||
inline void TrivialFactorWeightFst<A, F>::InitStateIterator(
|
||||
StateIteratorData<A> *data) const {
|
||||
data->base = new StateIterator< TrivialFactorWeightFst<A, F> >(*this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user