Initial commit: FunASR Speech Recognition Toolkit
Update API Documentation / build-api-docs (push) Has been cancelled
Update API Documentation / build-api-docs (push) Has been cancelled
Add complete FunASR codebase including models, runtime, and documentation.
This commit is contained in:
freedakgmail
@@ -0,0 +1,20 @@
|
||||
|
||||
|
||||
all:
|
||||
|
||||
|
||||
include ../kaldi.mk
|
||||
|
||||
TESTFILES = const-integer-set-test stl-utils-test text-utils-test \
|
||||
edit-distance-test hash-list-test kaldi-io-test parse-options-test \
|
||||
kaldi-table-test simple-options-test kaldi-thread-test
|
||||
|
||||
OBJFILES = text-utils.o kaldi-io.o kaldi-holder.o kaldi-table.o \
|
||||
parse-options.o simple-options.o simple-io-funcs.o \
|
||||
kaldi-semaphore.o kaldi-thread.o
|
||||
|
||||
LIBNAME = kaldi-util
|
||||
|
||||
ADDLIBS = ../matrix/kaldi-matrix.a ../base/kaldi-base.a
|
||||
|
||||
include ../makefiles/default_rules.mk
|
||||
@@ -0,0 +1,994 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// This is a modified version of the std::basic_filebuf from libc++
|
||||
// (http://libcxx.llvm.org/).
|
||||
// It allows one to create basic_filebuf from an existing FILE* handle or file
|
||||
// descriptor.
|
||||
//
|
||||
// This file is dual licensed under the MIT and the University of Illinois Open
|
||||
// Source License licenses. See LICENSE.TXT for details (included at the
|
||||
// bottom).
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
#ifndef KALDI_UTIL_BASIC_FILEBUF_H_
|
||||
#define KALDI_UTIL_BASIC_FILEBUF_H_
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
#include <fstream>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <algorithm>
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
namespace kaldi {
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <typename CharT, typename Traits = std::char_traits<CharT> >
|
||||
class basic_filebuf : public std::basic_streambuf<CharT, Traits> {
|
||||
public:
|
||||
typedef CharT char_type;
|
||||
typedef Traits traits_type;
|
||||
typedef typename traits_type::int_type int_type;
|
||||
typedef typename traits_type::pos_type pos_type;
|
||||
typedef typename traits_type::off_type off_type;
|
||||
typedef typename traits_type::state_type state_type;
|
||||
|
||||
basic_filebuf();
|
||||
basic_filebuf(basic_filebuf&& rhs);
|
||||
virtual ~basic_filebuf();
|
||||
|
||||
basic_filebuf& operator=(basic_filebuf&& rhs);
|
||||
void swap(basic_filebuf& rhs);
|
||||
|
||||
bool is_open() const;
|
||||
basic_filebuf* open(const char* s, std::ios_base::openmode mode);
|
||||
basic_filebuf* open(const std::string& s, std::ios_base::openmode mode);
|
||||
basic_filebuf* open(int fd, std::ios_base::openmode mode);
|
||||
basic_filebuf* open(FILE* f, std::ios_base::openmode mode);
|
||||
basic_filebuf* close();
|
||||
|
||||
FILE* file() { return this->_M_file; }
|
||||
int fd() { return fileno(this->_M_file); }
|
||||
|
||||
protected:
|
||||
int_type underflow() override;
|
||||
int_type pbackfail(int_type c = traits_type::eof()) override;
|
||||
int_type overflow(int_type c = traits_type::eof()) override;
|
||||
std::basic_streambuf<char_type, traits_type>*
|
||||
setbuf(char_type* s, std::streamsize n) override;
|
||||
pos_type seekoff(off_type off, std::ios_base::seekdir way,
|
||||
std::ios_base::openmode wch =
|
||||
std::ios_base::in | std::ios_base::out) override;
|
||||
pos_type seekpos(pos_type sp,
|
||||
std::ios_base::openmode wch =
|
||||
std::ios_base::in | std::ios_base::out) override;
|
||||
int sync() override;
|
||||
void imbue(const std::locale& loc) override;
|
||||
|
||||
protected:
|
||||
char* _M_extbuf;
|
||||
const char* _M_extbufnext;
|
||||
const char* _M_extbufend;
|
||||
char _M_extbuf_min[8];
|
||||
size_t _M_ebs;
|
||||
char_type* _M_intbuf;
|
||||
size_t _M_ibs;
|
||||
FILE* _M_file;
|
||||
const std::codecvt<char_type, char, state_type>* _M_cv;
|
||||
state_type _M_st;
|
||||
state_type _M_st_last;
|
||||
std::ios_base::openmode _M_om;
|
||||
std::ios_base::openmode _M_cm;
|
||||
bool _M_owns_eb;
|
||||
bool _M_owns_ib;
|
||||
bool _M_always_noconv;
|
||||
|
||||
const char* _M_get_mode(std::ios_base::openmode mode);
|
||||
bool _M_read_mode();
|
||||
void _M_write_mode();
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
basic_filebuf<CharT, Traits>::basic_filebuf()
|
||||
: _M_extbuf(nullptr),
|
||||
_M_extbufnext(nullptr),
|
||||
_M_extbufend(nullptr),
|
||||
_M_ebs(0),
|
||||
_M_intbuf(nullptr),
|
||||
_M_ibs(0),
|
||||
_M_file(nullptr),
|
||||
_M_cv(nullptr),
|
||||
_M_st(),
|
||||
_M_st_last(),
|
||||
_M_om(std::ios_base::openmode(0)),
|
||||
_M_cm(std::ios_base::openmode(0)),
|
||||
_M_owns_eb(false),
|
||||
_M_owns_ib(false),
|
||||
_M_always_noconv(false) {
|
||||
if (std::has_facet<std::codecvt<char_type, char, state_type> >
|
||||
(this->getloc())) {
|
||||
_M_cv = &std::use_facet<std::codecvt<char_type, char, state_type> >
|
||||
(this->getloc());
|
||||
_M_always_noconv = _M_cv->always_noconv();
|
||||
}
|
||||
setbuf(0, 4096);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
basic_filebuf<CharT, Traits>::basic_filebuf(basic_filebuf&& rhs)
|
||||
: std::basic_streambuf<CharT, Traits>(rhs) {
|
||||
if (rhs._M_extbuf == rhs._M_extbuf_min) {
|
||||
_M_extbuf = _M_extbuf_min;
|
||||
_M_extbufnext = _M_extbuf + (rhs._M_extbufnext - rhs._M_extbuf);
|
||||
_M_extbufend = _M_extbuf + (rhs._M_extbufend - rhs._M_extbuf);
|
||||
} else {
|
||||
_M_extbuf = rhs._M_extbuf;
|
||||
_M_extbufnext = rhs._M_extbufnext;
|
||||
_M_extbufend = rhs._M_extbufend;
|
||||
}
|
||||
_M_ebs = rhs._M_ebs;
|
||||
_M_intbuf = rhs._M_intbuf;
|
||||
_M_ibs = rhs._M_ibs;
|
||||
_M_file = rhs._M_file;
|
||||
_M_cv = rhs._M_cv;
|
||||
_M_st = rhs._M_st;
|
||||
_M_st_last = rhs._M_st_last;
|
||||
_M_om = rhs._M_om;
|
||||
_M_cm = rhs._M_cm;
|
||||
_M_owns_eb = rhs._M_owns_eb;
|
||||
_M_owns_ib = rhs._M_owns_ib;
|
||||
_M_always_noconv = rhs._M_always_noconv;
|
||||
if (rhs.pbase()) {
|
||||
if (rhs.pbase() == rhs._M_intbuf)
|
||||
this->setp(_M_intbuf, _M_intbuf + (rhs. epptr() - rhs.pbase()));
|
||||
else
|
||||
this->setp(reinterpret_cast<char_type*>(_M_extbuf),
|
||||
reinterpret_cast<char_type*>(_M_extbuf)
|
||||
+ (rhs. epptr() - rhs.pbase()));
|
||||
this->pbump(rhs. pptr() - rhs.pbase());
|
||||
} else if (rhs.eback()) {
|
||||
if (rhs.eback() == rhs._M_intbuf)
|
||||
this->setg(_M_intbuf, _M_intbuf + (rhs.gptr() - rhs.eback()),
|
||||
_M_intbuf + (rhs.egptr() - rhs.eback()));
|
||||
else
|
||||
this->setg(reinterpret_cast<char_type*>(_M_extbuf),
|
||||
reinterpret_cast<char_type*>(_M_extbuf) +
|
||||
(rhs.gptr() - rhs.eback()),
|
||||
reinterpret_cast<char_type*>(_M_extbuf) +
|
||||
(rhs.egptr() - rhs.eback()));
|
||||
}
|
||||
rhs._M_extbuf = nullptr;
|
||||
rhs._M_extbufnext = nullptr;
|
||||
rhs._M_extbufend = nullptr;
|
||||
rhs._M_ebs = 0;
|
||||
rhs._M_intbuf = nullptr;
|
||||
rhs._M_ibs = 0;
|
||||
rhs._M_file = nullptr;
|
||||
rhs._M_st = state_type();
|
||||
rhs._M_st_last = state_type();
|
||||
rhs._M_om = std::ios_base::openmode(0);
|
||||
rhs._M_cm = std::ios_base::openmode(0);
|
||||
rhs._M_owns_eb = false;
|
||||
rhs._M_owns_ib = false;
|
||||
rhs.setg(0, 0, 0);
|
||||
rhs.setp(0, 0);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
inline
|
||||
basic_filebuf<CharT, Traits>&
|
||||
basic_filebuf<CharT, Traits>::operator=(basic_filebuf&& rhs) {
|
||||
close();
|
||||
swap(rhs);
|
||||
return *this;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
basic_filebuf<CharT, Traits>::~basic_filebuf() {
|
||||
// try
|
||||
// {
|
||||
// close();
|
||||
// }
|
||||
// catch (...)
|
||||
// {
|
||||
// }
|
||||
if (_M_owns_eb)
|
||||
delete [] _M_extbuf;
|
||||
if (_M_owns_ib)
|
||||
delete [] _M_intbuf;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
void
|
||||
basic_filebuf<CharT, Traits>::swap(basic_filebuf& rhs) {
|
||||
std::basic_streambuf<char_type, traits_type>::swap(rhs);
|
||||
if (_M_extbuf != _M_extbuf_min && rhs._M_extbuf != rhs._M_extbuf_min) {
|
||||
std::swap(_M_extbuf, rhs._M_extbuf);
|
||||
std::swap(_M_extbufnext, rhs._M_extbufnext);
|
||||
std::swap(_M_extbufend, rhs._M_extbufend);
|
||||
} else {
|
||||
ptrdiff_t ln = _M_extbufnext - _M_extbuf;
|
||||
ptrdiff_t le = _M_extbufend - _M_extbuf;
|
||||
ptrdiff_t rn = rhs._M_extbufnext - rhs._M_extbuf;
|
||||
ptrdiff_t re = rhs._M_extbufend - rhs._M_extbuf;
|
||||
if (_M_extbuf == _M_extbuf_min && rhs._M_extbuf != rhs._M_extbuf_min) {
|
||||
_M_extbuf = rhs._M_extbuf;
|
||||
rhs._M_extbuf = rhs._M_extbuf_min;
|
||||
} else if (_M_extbuf != _M_extbuf_min &&
|
||||
rhs._M_extbuf == rhs._M_extbuf_min) {
|
||||
rhs._M_extbuf = _M_extbuf;
|
||||
_M_extbuf = _M_extbuf_min;
|
||||
}
|
||||
_M_extbufnext = _M_extbuf + rn;
|
||||
_M_extbufend = _M_extbuf + re;
|
||||
rhs._M_extbufnext = rhs._M_extbuf + ln;
|
||||
rhs._M_extbufend = rhs._M_extbuf + le;
|
||||
}
|
||||
std::swap(_M_ebs, rhs._M_ebs);
|
||||
std::swap(_M_intbuf, rhs._M_intbuf);
|
||||
std::swap(_M_ibs, rhs._M_ibs);
|
||||
std::swap(_M_file, rhs._M_file);
|
||||
std::swap(_M_cv, rhs._M_cv);
|
||||
std::swap(_M_st, rhs._M_st);
|
||||
std::swap(_M_st_last, rhs._M_st_last);
|
||||
std::swap(_M_om, rhs._M_om);
|
||||
std::swap(_M_cm, rhs._M_cm);
|
||||
std::swap(_M_owns_eb, rhs._M_owns_eb);
|
||||
std::swap(_M_owns_ib, rhs._M_owns_ib);
|
||||
std::swap(_M_always_noconv, rhs._M_always_noconv);
|
||||
if (this->eback() == reinterpret_cast<char_type*>(rhs._M_extbuf_min)) {
|
||||
ptrdiff_t n = this->gptr() - this->eback();
|
||||
ptrdiff_t e = this->egptr() - this->eback();
|
||||
this->setg(reinterpret_cast<char_type*>(_M_extbuf_min),
|
||||
reinterpret_cast<char_type*>(_M_extbuf_min) + n,
|
||||
reinterpret_cast<char_type*>(_M_extbuf_min) + e);
|
||||
} else if (this->pbase() ==
|
||||
reinterpret_cast<char_type*>(rhs._M_extbuf_min)) {
|
||||
ptrdiff_t n = this->pptr() - this->pbase();
|
||||
ptrdiff_t e = this->epptr() - this->pbase();
|
||||
this->setp(reinterpret_cast<char_type*>(_M_extbuf_min),
|
||||
reinterpret_cast<char_type*>(_M_extbuf_min) + e);
|
||||
this->pbump(n);
|
||||
}
|
||||
if (rhs.eback() == reinterpret_cast<char_type*>(_M_extbuf_min)) {
|
||||
ptrdiff_t n = rhs.gptr() - rhs.eback();
|
||||
ptrdiff_t e = rhs.egptr() - rhs.eback();
|
||||
rhs.setg(reinterpret_cast<char_type*>(rhs._M_extbuf_min),
|
||||
reinterpret_cast<char_type*>(rhs._M_extbuf_min) + n,
|
||||
reinterpret_cast<char_type*>(rhs._M_extbuf_min) + e);
|
||||
} else if (rhs.pbase() == reinterpret_cast<char_type*>(_M_extbuf_min)) {
|
||||
ptrdiff_t n = rhs.pptr() - rhs.pbase();
|
||||
ptrdiff_t e = rhs.epptr() - rhs.pbase();
|
||||
rhs.setp(reinterpret_cast<char_type*>(rhs._M_extbuf_min),
|
||||
reinterpret_cast<char_type*>(rhs._M_extbuf_min) + e);
|
||||
rhs.pbump(n);
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
inline
|
||||
void
|
||||
swap(basic_filebuf<CharT, Traits>& x, basic_filebuf<CharT, Traits>& y) {
|
||||
x.swap(y);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
inline
|
||||
bool
|
||||
basic_filebuf<CharT, Traits>::is_open() const {
|
||||
return _M_file != nullptr;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
const char* basic_filebuf<CharT, Traits>::
|
||||
_M_get_mode(std::ios_base::openmode mode) {
|
||||
switch ((mode & ~std::ios_base::ate) | 0) {
|
||||
case std::ios_base::out:
|
||||
case std::ios_base::out | std::ios_base::trunc:
|
||||
return "w";
|
||||
case std::ios_base::out | std::ios_base::app:
|
||||
case std::ios_base::app:
|
||||
return "a";
|
||||
break;
|
||||
case std::ios_base::in:
|
||||
return "r";
|
||||
case std::ios_base::in | std::ios_base::out:
|
||||
return "r+";
|
||||
case std::ios_base::in | std::ios_base::out | std::ios_base::trunc:
|
||||
return "w+";
|
||||
case std::ios_base::in | std::ios_base::out | std::ios_base::app:
|
||||
case std::ios_base::in | std::ios_base::app:
|
||||
return "a+";
|
||||
case std::ios_base::out | std::ios_base::binary:
|
||||
case std::ios_base::out | std::ios_base::trunc | std::ios_base::binary:
|
||||
return "wb";
|
||||
case std::ios_base::out | std::ios_base::app | std::ios_base::binary:
|
||||
case std::ios_base::app | std::ios_base::binary:
|
||||
return "ab";
|
||||
case std::ios_base::in | std::ios_base::binary:
|
||||
return "rb";
|
||||
case std::ios_base::in | std::ios_base::out | std::ios_base::binary:
|
||||
return "r+b";
|
||||
case std::ios_base::in | std::ios_base::out | std::ios_base::trunc |
|
||||
std::ios_base::binary:
|
||||
return "w+b";
|
||||
case std::ios_base::in | std::ios_base::out | std::ios_base::app |
|
||||
std::ios_base::binary:
|
||||
case std::ios_base::in | std::ios_base::app | std::ios_base::binary:
|
||||
return "a+b";
|
||||
default:
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
basic_filebuf<CharT, Traits>*
|
||||
basic_filebuf<CharT, Traits>::
|
||||
open(const char* s, std::ios_base::openmode mode) {
|
||||
basic_filebuf<CharT, Traits>* rt = nullptr;
|
||||
if (_M_file == nullptr) {
|
||||
const char* md= _M_get_mode(mode);
|
||||
if (md) {
|
||||
_M_file = fopen(s, md);
|
||||
if (_M_file) {
|
||||
rt = this;
|
||||
_M_om = mode;
|
||||
if (mode & std::ios_base::ate) {
|
||||
if (fseek(_M_file, 0, SEEK_END)) {
|
||||
fclose(_M_file);
|
||||
_M_file = nullptr;
|
||||
rt = nullptr;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rt;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
inline
|
||||
basic_filebuf<CharT, Traits>*
|
||||
basic_filebuf<CharT, Traits>::open(const std::string& s,
|
||||
std::ios_base::openmode mode) {
|
||||
return open(s.c_str(), mode);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
basic_filebuf<CharT, Traits>*
|
||||
basic_filebuf<CharT, Traits>::open(int fd, std::ios_base::openmode mode) {
|
||||
const char* md= this->_M_get_mode(mode);
|
||||
if (md) {
|
||||
this->_M_file= fdopen(fd, md);
|
||||
this->_M_om = mode;
|
||||
return this;
|
||||
} else {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
basic_filebuf<CharT, Traits>*
|
||||
basic_filebuf<CharT, Traits>::open(FILE* f, std::ios_base::openmode mode) {
|
||||
this->_M_file = f;
|
||||
this->_M_om = mode;
|
||||
return this;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
basic_filebuf<CharT, Traits>*
|
||||
basic_filebuf<CharT, Traits>::close() {
|
||||
basic_filebuf<CharT, Traits>* rt = nullptr;
|
||||
if (_M_file) {
|
||||
rt = this;
|
||||
std::unique_ptr<FILE, int(*)(FILE*)> h(_M_file, fclose);
|
||||
if (sync())
|
||||
rt = nullptr;
|
||||
if (fclose(h.release()) == 0)
|
||||
_M_file = nullptr;
|
||||
else
|
||||
rt = nullptr;
|
||||
}
|
||||
return rt;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
typename basic_filebuf<CharT, Traits>::int_type
|
||||
basic_filebuf<CharT, Traits>::underflow() {
|
||||
if (_M_file == nullptr)
|
||||
return traits_type::eof();
|
||||
bool initial = _M_read_mode();
|
||||
char_type buf;
|
||||
if (this->gptr() == nullptr)
|
||||
this->setg(&buf, &buf+1, &buf+1);
|
||||
const size_t unget_sz = initial ? 0 : std::
|
||||
min<size_t>((this->egptr() - this->eback()) / 2, 4);
|
||||
int_type c = traits_type::eof();
|
||||
if (this->gptr() == this->egptr()) {
|
||||
memmove(this->eback(), this->egptr() - unget_sz,
|
||||
unget_sz * sizeof(char_type));
|
||||
if (_M_always_noconv) {
|
||||
size_t nmemb = static_cast<size_t>
|
||||
(this->egptr() - this->eback() - unget_sz);
|
||||
nmemb = fread(this->eback() + unget_sz, 1, nmemb, _M_file);
|
||||
if (nmemb != 0) {
|
||||
this->setg(this->eback(),
|
||||
this->eback() + unget_sz,
|
||||
this->eback() + unget_sz + nmemb);
|
||||
c = traits_type::to_int_type(*this->gptr());
|
||||
}
|
||||
} else {
|
||||
memmove(_M_extbuf, _M_extbufnext, _M_extbufend - _M_extbufnext);
|
||||
_M_extbufnext = _M_extbuf + (_M_extbufend - _M_extbufnext);
|
||||
_M_extbufend = _M_extbuf +
|
||||
(_M_extbuf == _M_extbuf_min ? sizeof(_M_extbuf_min) : _M_ebs);
|
||||
size_t nmemb = std::min(static_cast<size_t>(_M_ibs - unget_sz),
|
||||
static_cast<size_t>
|
||||
(_M_extbufend - _M_extbufnext));
|
||||
std::codecvt_base::result r;
|
||||
_M_st_last = _M_st;
|
||||
size_t nr = fread(
|
||||
reinterpret_cast<void*>(const_cast<char_type*>(_M_extbufnext)),
|
||||
1, nmemb, _M_file);
|
||||
if (nr != 0) {
|
||||
if (!_M_cv)
|
||||
throw std::bad_cast();
|
||||
_M_extbufend = _M_extbufnext + nr;
|
||||
char_type* inext;
|
||||
r = _M_cv->in(_M_st, _M_extbuf, _M_extbufend, _M_extbufnext,
|
||||
this->eback() + unget_sz,
|
||||
this->eback() + _M_ibs, inext);
|
||||
if (r == std::codecvt_base::noconv) {
|
||||
this->setg(reinterpret_cast<char_type*>(_M_extbuf),
|
||||
reinterpret_cast<char_type*>(_M_extbuf),
|
||||
const_cast<char_type*>(_M_extbufend));
|
||||
c = traits_type::to_int_type(*this->gptr());
|
||||
} else if (inext != this->eback() + unget_sz) {
|
||||
this->setg(this->eback(), this->eback() + unget_sz, inext);
|
||||
c = traits_type::to_int_type(*this->gptr());
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
c = traits_type::to_int_type(*this->gptr());
|
||||
}
|
||||
if (this->eback() == &buf)
|
||||
this->setg(0, 0, 0);
|
||||
return c;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
typename basic_filebuf<CharT, Traits>::int_type
|
||||
basic_filebuf<CharT, Traits>::pbackfail(int_type c) {
|
||||
if (_M_file && this->eback() < this->gptr()) {
|
||||
if (traits_type::eq_int_type(c, traits_type::eof())) {
|
||||
this->gbump(-1);
|
||||
return traits_type::not_eof(c);
|
||||
}
|
||||
if ((_M_om & std::ios_base::out) ||
|
||||
traits_type::eq(traits_type::to_char_type(c), this->gptr()[-1])) {
|
||||
this->gbump(-1);
|
||||
*this->gptr() = traits_type::to_char_type(c);
|
||||
return c;
|
||||
}
|
||||
}
|
||||
return traits_type::eof();
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
typename basic_filebuf<CharT, Traits>::int_type
|
||||
basic_filebuf<CharT, Traits>::overflow(int_type c) {
|
||||
if (_M_file == nullptr)
|
||||
return traits_type::eof();
|
||||
_M_write_mode();
|
||||
char_type buf;
|
||||
char_type* pb_save = this->pbase();
|
||||
char_type* epb_save = this->epptr();
|
||||
if (!traits_type::eq_int_type(c, traits_type::eof())) {
|
||||
if (this->pptr() == nullptr)
|
||||
this->setp(&buf, &buf+1);
|
||||
*this->pptr() = traits_type::to_char_type(c);
|
||||
this->pbump(1);
|
||||
}
|
||||
if (this->pptr() != this->pbase()) {
|
||||
if (_M_always_noconv) {
|
||||
size_t nmemb = static_cast<size_t>(this->pptr() - this->pbase());
|
||||
if (fwrite(this->pbase(), sizeof(char_type),
|
||||
nmemb, _M_file) != nmemb)
|
||||
return traits_type::eof();
|
||||
} else {
|
||||
char* extbe = _M_extbuf;
|
||||
std::codecvt_base::result r;
|
||||
do {
|
||||
if (!_M_cv)
|
||||
throw std::bad_cast();
|
||||
const char_type* e;
|
||||
r = _M_cv->out(_M_st, this->pbase(), this->pptr(), e,
|
||||
_M_extbuf, _M_extbuf + _M_ebs, extbe);
|
||||
if (e == this->pbase())
|
||||
return traits_type::eof();
|
||||
if (r == std::codecvt_base::noconv) {
|
||||
size_t nmemb = static_cast<size_t>
|
||||
(this->pptr() - this->pbase());
|
||||
if (fwrite(this->pbase(), 1, nmemb, _M_file) != nmemb)
|
||||
return traits_type::eof();
|
||||
} else if (r == std::codecvt_base::ok ||
|
||||
r == std::codecvt_base::partial) {
|
||||
size_t nmemb = static_cast<size_t>(extbe - _M_extbuf);
|
||||
if (fwrite(_M_extbuf, 1, nmemb, _M_file) != nmemb)
|
||||
return traits_type::eof();
|
||||
if (r == std::codecvt_base::partial) {
|
||||
this->setp(const_cast<char_type*>(e),
|
||||
this->pptr());
|
||||
this->pbump(this->epptr() - this->pbase());
|
||||
}
|
||||
} else {
|
||||
return traits_type::eof();
|
||||
}
|
||||
} while (r == std::codecvt_base::partial);
|
||||
}
|
||||
this->setp(pb_save, epb_save);
|
||||
}
|
||||
return traits_type::not_eof(c);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
std::basic_streambuf<CharT, Traits>*
|
||||
basic_filebuf<CharT, Traits>::setbuf(char_type* s, std::streamsize n) {
|
||||
this->setg(0, 0, 0);
|
||||
this->setp(0, 0);
|
||||
if (_M_owns_eb)
|
||||
delete [] _M_extbuf;
|
||||
if (_M_owns_ib)
|
||||
delete [] _M_intbuf;
|
||||
_M_ebs = n;
|
||||
if (_M_ebs > sizeof(_M_extbuf_min)) {
|
||||
if (_M_always_noconv && s) {
|
||||
_M_extbuf = reinterpret_cast<char*>(s);
|
||||
_M_owns_eb = false;
|
||||
} else {
|
||||
_M_extbuf = new char[_M_ebs];
|
||||
_M_owns_eb = true;
|
||||
}
|
||||
} else {
|
||||
_M_extbuf = _M_extbuf_min;
|
||||
_M_ebs = sizeof(_M_extbuf_min);
|
||||
_M_owns_eb = false;
|
||||
}
|
||||
if (!_M_always_noconv) {
|
||||
_M_ibs = std::max<std::streamsize>(n, sizeof(_M_extbuf_min));
|
||||
if (s && _M_ibs >= sizeof(_M_extbuf_min)) {
|
||||
_M_intbuf = s;
|
||||
_M_owns_ib = false;
|
||||
} else {
|
||||
_M_intbuf = new char_type[_M_ibs];
|
||||
_M_owns_ib = true;
|
||||
}
|
||||
} else {
|
||||
_M_ibs = 0;
|
||||
_M_intbuf = 0;
|
||||
_M_owns_ib = false;
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
typename basic_filebuf<CharT, Traits>::pos_type
|
||||
basic_filebuf<CharT, Traits>::seekoff(off_type off, std::ios_base::seekdir way,
|
||||
std::ios_base::openmode) {
|
||||
if (!_M_cv)
|
||||
throw std::bad_cast();
|
||||
int width = _M_cv->encoding();
|
||||
if (_M_file == nullptr || (width <= 0 && off != 0) || sync())
|
||||
return pos_type(off_type(-1));
|
||||
// width > 0 || off == 0
|
||||
int whence;
|
||||
switch (way) {
|
||||
case std::ios_base::beg:
|
||||
whence = SEEK_SET;
|
||||
break;
|
||||
case std::ios_base::cur:
|
||||
whence = SEEK_CUR;
|
||||
break;
|
||||
case std::ios_base::end:
|
||||
whence = SEEK_END;
|
||||
break;
|
||||
default:
|
||||
return pos_type(off_type(-1));
|
||||
}
|
||||
#if _WIN32
|
||||
if (fseek(_M_file, width > 0 ? width * off : 0, whence))
|
||||
return pos_type(off_type(-1));
|
||||
pos_type r = ftell(_M_file);
|
||||
#else
|
||||
if (fseeko(_M_file, width > 0 ? width * off : 0, whence))
|
||||
return pos_type(off_type(-1));
|
||||
pos_type r = ftello(_M_file);
|
||||
#endif
|
||||
r.state(_M_st);
|
||||
return r;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
typename basic_filebuf<CharT, Traits>::pos_type
|
||||
basic_filebuf<CharT, Traits>::seekpos(pos_type sp, std::ios_base::openmode) {
|
||||
if (_M_file == nullptr || sync())
|
||||
return pos_type(off_type(-1));
|
||||
#if _WIN32
|
||||
if (fseek(_M_file, sp, SEEK_SET))
|
||||
return pos_type(off_type(-1));
|
||||
#else
|
||||
if (fseeko(_M_file, sp, SEEK_SET))
|
||||
return pos_type(off_type(-1));
|
||||
#endif
|
||||
_M_st = sp.state();
|
||||
return sp;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
int
|
||||
basic_filebuf<CharT, Traits>::sync() {
|
||||
if (_M_file == nullptr)
|
||||
return 0;
|
||||
if (!_M_cv)
|
||||
throw std::bad_cast();
|
||||
if (_M_cm & std::ios_base::out) {
|
||||
if (this->pptr() != this->pbase())
|
||||
if (overflow() == traits_type::eof())
|
||||
return -1;
|
||||
std::codecvt_base::result r;
|
||||
do {
|
||||
char* extbe;
|
||||
r = _M_cv->unshift(_M_st, _M_extbuf, _M_extbuf + _M_ebs, extbe);
|
||||
size_t nmemb = static_cast<size_t>(extbe - _M_extbuf);
|
||||
if (fwrite(_M_extbuf, 1, nmemb, _M_file) != nmemb)
|
||||
return -1;
|
||||
} while (r == std::codecvt_base::partial);
|
||||
if (r == std::codecvt_base::error)
|
||||
return -1;
|
||||
if (fflush(_M_file))
|
||||
return -1;
|
||||
} else if (_M_cm & std::ios_base::in) {
|
||||
off_type c;
|
||||
state_type state = _M_st_last;
|
||||
bool update_st = false;
|
||||
if (_M_always_noconv) {
|
||||
c = this->egptr() - this->gptr();
|
||||
} else {
|
||||
int width = _M_cv->encoding();
|
||||
c = _M_extbufend - _M_extbufnext;
|
||||
if (width > 0) {
|
||||
c += width * (this->egptr() - this->gptr());
|
||||
} else {
|
||||
if (this->gptr() != this->egptr()) {
|
||||
const int off = _M_cv->length(state, _M_extbuf,
|
||||
_M_extbufnext,
|
||||
this->gptr() - this->eback());
|
||||
c += _M_extbufnext - _M_extbuf - off;
|
||||
update_st = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
#if _WIN32
|
||||
if (fseek(_M_file_, -c, SEEK_CUR))
|
||||
return -1;
|
||||
#else
|
||||
if (fseeko(_M_file, -c, SEEK_CUR))
|
||||
return -1;
|
||||
#endif
|
||||
if (update_st)
|
||||
_M_st = state;
|
||||
_M_extbufnext = _M_extbufend = _M_extbuf;
|
||||
this->setg(0, 0, 0);
|
||||
_M_cm = std::ios_base::openmode(0);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
void
|
||||
basic_filebuf<CharT, Traits>::imbue(const std::locale& loc) {
|
||||
sync();
|
||||
_M_cv = &std::use_facet<std::codecvt<char_type, char, state_type> >(loc);
|
||||
bool old_anc = _M_always_noconv;
|
||||
_M_always_noconv = _M_cv->always_noconv();
|
||||
if (old_anc != _M_always_noconv) {
|
||||
this->setg(0, 0, 0);
|
||||
this->setp(0, 0);
|
||||
// invariant, char_type is char, else we couldn't get here
|
||||
// need to dump _M_intbuf
|
||||
if (_M_always_noconv) {
|
||||
if (_M_owns_eb)
|
||||
delete [] _M_extbuf;
|
||||
_M_owns_eb = _M_owns_ib;
|
||||
_M_ebs = _M_ibs;
|
||||
_M_extbuf = reinterpret_cast<char*>(_M_intbuf);
|
||||
_M_ibs = 0;
|
||||
_M_intbuf = nullptr;
|
||||
_M_owns_ib = false;
|
||||
} else { // need to obtain an _M_intbuf.
|
||||
// If _M_extbuf is user-supplied, use it, else new _M_intbuf
|
||||
if (!_M_owns_eb && _M_extbuf != _M_extbuf_min) {
|
||||
_M_ibs = _M_ebs;
|
||||
_M_intbuf = reinterpret_cast<char_type*>(_M_extbuf);
|
||||
_M_owns_ib = false;
|
||||
_M_extbuf = new char[_M_ebs];
|
||||
_M_owns_eb = true;
|
||||
} else {
|
||||
_M_ibs = _M_ebs;
|
||||
_M_intbuf = new char_type[_M_ibs];
|
||||
_M_owns_ib = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
bool
|
||||
basic_filebuf<CharT, Traits>::_M_read_mode() {
|
||||
if (!(_M_cm & std::ios_base::in)) {
|
||||
this->setp(0, 0);
|
||||
if (_M_always_noconv)
|
||||
this->setg(reinterpret_cast<char_type*>(_M_extbuf),
|
||||
reinterpret_cast<char_type*>(_M_extbuf) + _M_ebs,
|
||||
reinterpret_cast<char_type*>(_M_extbuf) + _M_ebs);
|
||||
else
|
||||
this->setg(_M_intbuf, _M_intbuf + _M_ibs, _M_intbuf + _M_ibs);
|
||||
_M_cm = std::ios_base::in;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
template <class CharT, class Traits>
|
||||
void
|
||||
basic_filebuf<CharT, Traits>::_M_write_mode() {
|
||||
if (!(_M_cm & std::ios_base::out)) {
|
||||
this->setg(0, 0, 0);
|
||||
if (_M_ebs > sizeof(_M_extbuf_min)) {
|
||||
if (_M_always_noconv)
|
||||
this->setp(reinterpret_cast<char_type*>(_M_extbuf),
|
||||
reinterpret_cast<char_type*>(_M_extbuf) +
|
||||
(_M_ebs - 1));
|
||||
else
|
||||
this->setp(_M_intbuf, _M_intbuf + (_M_ibs - 1));
|
||||
} else {
|
||||
this->setp(0, 0);
|
||||
}
|
||||
_M_cm = std::ios_base::out;
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
#endif // KALDI_UTIL_BASIC_FILEBUF_H_
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/*
|
||||
* ============================================================================
|
||||
* libc++ License
|
||||
* ============================================================================
|
||||
*
|
||||
* The libc++ library is dual licensed under both the University of Illinois
|
||||
* "BSD-Like" license and the MIT license. As a user of this code you may
|
||||
* choose to use it under either license. As a contributor, you agree to allow
|
||||
* your code to be used under both.
|
||||
*
|
||||
* Full text of the relevant licenses is included below.
|
||||
*
|
||||
* ============================================================================
|
||||
*
|
||||
* University of Illinois/NCSA
|
||||
* Open Source License
|
||||
*
|
||||
* Copyright (c) 2009-2014 by the contributors listed in CREDITS.TXT (included below)
|
||||
*
|
||||
* All rights reserved.
|
||||
*
|
||||
* Developed by:
|
||||
*
|
||||
* LLVM Team
|
||||
*
|
||||
* University of Illinois at Urbana-Champaign
|
||||
*
|
||||
* http://llvm.org
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal with
|
||||
* the Software without restriction, including without limitation the rights to
|
||||
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is furnished to do
|
||||
* so, subject to the following conditions:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimers.
|
||||
*
|
||||
* * Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimers in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* * Neither the names of the LLVM Team, University of Illinois at
|
||||
* Urbana-Champaign, nor the names of its contributors may be used to
|
||||
* endorse or promote products derived from this Software without specific
|
||||
* prior written permission.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
|
||||
* SOFTWARE.
|
||||
*
|
||||
* ==============================================================================
|
||||
*
|
||||
* Copyright (c) 2009-2014 by the contributors listed in CREDITS.TXT (included below)
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
* THE SOFTWARE.
|
||||
*
|
||||
* ==============================================================================
|
||||
*
|
||||
* This file is a partial list of people who have contributed to the LLVM/libc++
|
||||
* project. If you have contributed a patch or made some other contribution to
|
||||
* LLVM/libc++, please submit a patch to this file to add yourself, and it will be
|
||||
* done!
|
||||
*
|
||||
* The list is sorted by surname and formatted to allow easy grepping and
|
||||
* beautification by scripts. The fields are: name (N), email (E), web-address
|
||||
* (W), PGP key ID and fingerprint (P), description (D), and snail-mail address
|
||||
* (S).
|
||||
*
|
||||
* N: Saleem Abdulrasool
|
||||
* E: compnerd@compnerd.org
|
||||
* D: Minor patches and Linux fixes.
|
||||
*
|
||||
* N: Dimitry Andric
|
||||
* E: dimitry@andric.com
|
||||
* D: Visibility fixes, minor FreeBSD portability patches.
|
||||
*
|
||||
* N: Holger Arnold
|
||||
* E: holgerar@gmail.com
|
||||
* D: Minor fix.
|
||||
*
|
||||
* N: Ruben Van Boxem
|
||||
* E: vanboxem dot ruben at gmail dot com
|
||||
* D: Initial Windows patches.
|
||||
*
|
||||
* N: David Chisnall
|
||||
* E: theraven at theravensnest dot org
|
||||
* D: FreeBSD and Solaris ports, libcxxrt support, some atomics work.
|
||||
*
|
||||
* N: Marshall Clow
|
||||
* E: mclow.lists@gmail.com
|
||||
* E: marshall@idio.com
|
||||
* D: C++14 support, patches and bug fixes.
|
||||
*
|
||||
* N: Bill Fisher
|
||||
* E: william.w.fisher@gmail.com
|
||||
* D: Regex bug fixes.
|
||||
*
|
||||
* N: Matthew Dempsky
|
||||
* E: matthew@dempsky.org
|
||||
* D: Minor patches and bug fixes.
|
||||
*
|
||||
* N: Google Inc.
|
||||
* D: Copyright owner and contributor of the CityHash algorithm
|
||||
*
|
||||
* N: Howard Hinnant
|
||||
* E: hhinnant@apple.com
|
||||
* D: Architect and primary author of libc++
|
||||
*
|
||||
* N: Hyeon-bin Jeong
|
||||
* E: tuhertz@gmail.com
|
||||
* D: Minor patches and bug fixes.
|
||||
*
|
||||
* N: Argyrios Kyrtzidis
|
||||
* E: kyrtzidis@apple.com
|
||||
* D: Bug fixes.
|
||||
*
|
||||
* N: Bruce Mitchener, Jr.
|
||||
* E: bruce.mitchener@gmail.com
|
||||
* D: Emscripten-related changes.
|
||||
*
|
||||
* N: Michel Morin
|
||||
* E: mimomorin@gmail.com
|
||||
* D: Minor patches to is_convertible.
|
||||
*
|
||||
* N: Andrew Morrow
|
||||
* E: andrew.c.morrow@gmail.com
|
||||
* D: Minor patches and Linux fixes.
|
||||
*
|
||||
* N: Arvid Picciani
|
||||
* E: aep at exys dot org
|
||||
* D: Minor patches and musl port.
|
||||
*
|
||||
* N: Bjorn Reese
|
||||
* E: breese@users.sourceforge.net
|
||||
* D: Initial regex prototype
|
||||
*
|
||||
* N: Nico Rieck
|
||||
* E: nico.rieck@gmail.com
|
||||
* D: Windows fixes
|
||||
*
|
||||
* N: Jonathan Sauer
|
||||
* D: Minor patches, mostly related to constexpr
|
||||
*
|
||||
* N: Craig Silverstein
|
||||
* E: csilvers@google.com
|
||||
* D: Implemented Cityhash as the string hash function on 64-bit machines
|
||||
*
|
||||
* N: Richard Smith
|
||||
* D: Minor patches.
|
||||
*
|
||||
* N: Joerg Sonnenberger
|
||||
* E: joerg@NetBSD.org
|
||||
* D: NetBSD port.
|
||||
*
|
||||
* N: Stephan Tolksdorf
|
||||
* E: st@quanttec.com
|
||||
* D: Minor <atomic> fix
|
||||
*
|
||||
* N: Michael van der Westhuizen
|
||||
* E: r1mikey at gmail dot com
|
||||
*
|
||||
* N: Klaas de Vries
|
||||
* E: klaas at klaasgaaf dot nl
|
||||
* D: Minor bug fix.
|
||||
*
|
||||
* N: Zhang Xiongpang
|
||||
* E: zhangxiongpang@gmail.com
|
||||
* D: Minor patches and bug fixes.
|
||||
*
|
||||
* N: Xing Xue
|
||||
* E: xingxue@ca.ibm.com
|
||||
* D: AIX port
|
||||
*
|
||||
* N: Zhihao Yuan
|
||||
* E: lichray@gmail.com
|
||||
* D: Standard compatibility fixes.
|
||||
*
|
||||
* N: Jeffrey Yasskin
|
||||
* E: jyasskin@gmail.com
|
||||
* E: jyasskin@google.com
|
||||
* D: Linux fixes.
|
||||
*/
|
||||
@@ -0,0 +1,91 @@
|
||||
// util/const-integer-set-inl.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#ifndef KALDI_UTIL_CONST_INTEGER_SET_INL_H_
|
||||
#define KALDI_UTIL_CONST_INTEGER_SET_INL_H_
|
||||
|
||||
// Do not include this file directly. It is included by const-integer-set.h
|
||||
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
template<class I>
|
||||
void ConstIntegerSet<I>::InitInternal() {
|
||||
KALDI_ASSERT_IS_INTEGER_TYPE(I);
|
||||
quick_set_.clear(); // just in case we previously had data.
|
||||
if (slow_set_.size() == 0) {
|
||||
lowest_member_=(I) 1;
|
||||
highest_member_=(I) 0;
|
||||
contiguous_ = false;
|
||||
quick_ = false;
|
||||
} else {
|
||||
lowest_member_ = slow_set_.front();
|
||||
highest_member_ = slow_set_.back();
|
||||
size_t range = highest_member_ + 1 - lowest_member_;
|
||||
if (range == slow_set_.size()) {
|
||||
contiguous_ = true;
|
||||
quick_= false;
|
||||
} else {
|
||||
contiguous_ = false;
|
||||
// If it would be more compact to store as bool
|
||||
if (range < slow_set_.size() * 8 * sizeof(I)) {
|
||||
// (assuming 1 bit per element)...
|
||||
quick_set_.resize(range, false);
|
||||
for (size_t i = 0;i < slow_set_.size();i++)
|
||||
quick_set_[slow_set_[i] - lowest_member_] = true;
|
||||
quick_ = true;
|
||||
} else {
|
||||
quick_ = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<class I>
|
||||
int ConstIntegerSet<I>::count(I i) const {
|
||||
if (i < lowest_member_ || i > highest_member_) {
|
||||
return 0;
|
||||
} else {
|
||||
if (contiguous_) return true;
|
||||
if (quick_) {
|
||||
return (quick_set_[i-lowest_member_] ? 1 : 0);
|
||||
} else {
|
||||
bool ans = std::binary_search(slow_set_.begin(), slow_set_.end(), i);
|
||||
return (ans ? 1 : 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<class I>
|
||||
void ConstIntegerSet<I>::Write(std::ostream &os, bool binary) const {
|
||||
WriteIntegerVector(os, binary, slow_set_);
|
||||
}
|
||||
|
||||
template<class I>
|
||||
void ConstIntegerSet<I>::Read(std::istream &is, bool binary) {
|
||||
ReadIntegerVector(is, binary, &slow_set_);
|
||||
InitInternal();
|
||||
}
|
||||
|
||||
|
||||
|
||||
} // end namespace kaldi
|
||||
|
||||
#endif // KALDI_UTIL_CONST_INTEGER_SET_INL_H_
|
||||
@@ -0,0 +1,145 @@
|
||||
// util/const-integer-set-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#include "util/const-integer-set.h"
|
||||
#include <set> // for baseline.
|
||||
#include <cassert>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include "util/kaldi-io.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
template<class Int> void TestSetOfNumbers(bool binary) {
|
||||
std::set<Int> baseline_set;
|
||||
size_t n_in_set = (Rand() % 3) * 50 + (Rand() % 4); // may be less than this.
|
||||
size_t max = (Int) (Rand() % 100) + 1;
|
||||
for (size_t i = 0; i < n_in_set; i++) {
|
||||
Int to_add((Int) (Rand() % max));
|
||||
baseline_set.insert(to_add);
|
||||
}
|
||||
|
||||
std::vector<Int> vector_set;
|
||||
for (typename std::set<Int>::iterator iter = baseline_set.begin();
|
||||
iter!= baseline_set.end();iter++)
|
||||
vector_set.push_back(*iter);
|
||||
if (vector_set.size() != 0) {
|
||||
for (size_t i = 0;i < 10;i++) // randomize order.
|
||||
std::swap(vector_set[Rand()%vector_set.size()],
|
||||
vector_set[Rand()%vector_set.size()]);
|
||||
}
|
||||
|
||||
ConstIntegerSet<Int> my_set1(baseline_set);
|
||||
|
||||
ConstIntegerSet<Int> my_set2(vector_set);
|
||||
|
||||
ConstIntegerSet<Int> my_set3;
|
||||
my_set3.Init(baseline_set);
|
||||
|
||||
ConstIntegerSet<Int> my_set4;
|
||||
my_set4.Init(vector_set);
|
||||
|
||||
{
|
||||
my_set4.Write(Output("tmpf", binary).Stream(), binary);
|
||||
}
|
||||
|
||||
ConstIntegerSet<Int> my_set5;
|
||||
{
|
||||
bool binary_in;
|
||||
Input ki("tmpf", &binary_in);
|
||||
my_set5.Read(ki.Stream(), binary_in);
|
||||
}
|
||||
|
||||
|
||||
// if (enable_iterators) {
|
||||
size_t sz = baseline_set.size(), sz1 = my_set1.size(), sz2 = my_set2.size(),
|
||||
sz3 = my_set3.size(), sz4 = my_set4.size(), sz5 = my_set5.size();
|
||||
KALDI_ASSERT(sz == sz1 && sz == sz2 && sz == sz3 && sz == sz4 && sz == sz5);
|
||||
// }
|
||||
for (size_t i = 0;i < 100;i++) {
|
||||
Int some_int;
|
||||
if (i%2 == 0 && vector_set.size() != 0)
|
||||
some_int = vector_set[Rand()%vector_set.size()];
|
||||
else
|
||||
some_int = Rand() % max;
|
||||
bool in_baseline = (baseline_set.count(some_int) != 0);
|
||||
bool in_my_set1 = (my_set1.count(some_int) != 0);
|
||||
bool in_my_set2 = (my_set2.count(some_int) != 0);
|
||||
bool in_my_set3 = (my_set3.count(some_int) != 0);
|
||||
bool in_my_set4 = (my_set4.count(some_int) != 0);
|
||||
bool in_my_set5 = (my_set5.count(some_int) != 0);
|
||||
|
||||
if (in_baseline) {
|
||||
KALDI_ASSERT(in_my_set1&&in_my_set2&&in_my_set3&&in_my_set4&&in_my_set5);
|
||||
} else {
|
||||
KALDI_ASSERT(!in_my_set1&&!in_my_set2&&!in_my_set3&&!in_my_set4&&
|
||||
!in_my_set5);
|
||||
}
|
||||
}
|
||||
|
||||
// if (enable_iterators) {
|
||||
typename std::set<Int>::iterator baseline_iter = baseline_set.begin();
|
||||
typename ConstIntegerSet<Int>::iterator my_iter1 = my_set1.begin();
|
||||
typename ConstIntegerSet<Int>::iterator my_iter2 = my_set2.begin();
|
||||
typename ConstIntegerSet<Int>::iterator my_iter3 = my_set3.begin();
|
||||
typename ConstIntegerSet<Int>::iterator my_iter4 = my_set4.begin();
|
||||
typename ConstIntegerSet<Int>::iterator my_iter5 = my_set5.begin();
|
||||
while (baseline_iter != baseline_set.end()) {
|
||||
KALDI_ASSERT(my_iter1 != my_set1.end());
|
||||
KALDI_ASSERT(my_iter2 != my_set2.end());
|
||||
KALDI_ASSERT(my_iter3 != my_set3.end());
|
||||
KALDI_ASSERT(my_iter4 != my_set4.end());
|
||||
KALDI_ASSERT(my_iter5 != my_set5.end());
|
||||
KALDI_ASSERT(*baseline_iter == *my_iter1);
|
||||
KALDI_ASSERT(*baseline_iter == *my_iter2);
|
||||
KALDI_ASSERT(*baseline_iter == *my_iter3);
|
||||
KALDI_ASSERT(*baseline_iter == *my_iter4);
|
||||
KALDI_ASSERT(*baseline_iter == *my_iter5);
|
||||
baseline_iter++;
|
||||
my_iter1++;
|
||||
my_iter2++;
|
||||
my_iter3++;
|
||||
my_iter4++;
|
||||
my_iter5++;
|
||||
}
|
||||
KALDI_ASSERT(my_iter1 == my_set1.end());
|
||||
KALDI_ASSERT(my_iter2 == my_set2.end());
|
||||
KALDI_ASSERT(my_iter3 == my_set3.end());
|
||||
KALDI_ASSERT(my_iter4 == my_set4.end());
|
||||
KALDI_ASSERT(my_iter5 == my_set5.end());
|
||||
// }
|
||||
}
|
||||
|
||||
} // end namespace kaldi
|
||||
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace kaldi;
|
||||
for (size_t i = 0;i < 10;i++) {
|
||||
TestSetOfNumbers<int>(Rand()%2);
|
||||
TestSetOfNumbers<unsigned int>(Rand()%2);
|
||||
TestSetOfNumbers<int16>(Rand()%2);
|
||||
TestSetOfNumbers<int16>(Rand()%2);
|
||||
TestSetOfNumbers<char>(Rand()%2);
|
||||
TestSetOfNumbers<unsigned char>(Rand()%2);
|
||||
}
|
||||
std::cout << "Test OK.\n";
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// util/const-integer-set.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#ifndef KALDI_UTIL_CONST_INTEGER_SET_H_
|
||||
#define KALDI_UTIL_CONST_INTEGER_SET_H_
|
||||
#include <vector>
|
||||
#include <set>
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <cassert>
|
||||
#include "util/stl-utils.h"
|
||||
|
||||
/* ConstIntegerSet is a way to efficiently test whether something is in a
|
||||
supplied set of integers. It can be initialized from a vector or set, but
|
||||
never changed after that. It either uses a sorted vector or an array of
|
||||
bool, depending on the input. It behaves like a const version of an STL set, with
|
||||
only a subset of the functionality, except all the member functions are
|
||||
upper-case.
|
||||
|
||||
Note that we could get rid of the member slow_set_, but we'd have to
|
||||
do more work to implement an iterator type. This would save memory.
|
||||
*/
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
template<class I> class ConstIntegerSet {
|
||||
public:
|
||||
ConstIntegerSet(): lowest_member_(1), highest_member_(0) { }
|
||||
|
||||
void Init(const std::vector<I> &input) {
|
||||
slow_set_ = input;
|
||||
SortAndUniq(&slow_set_);
|
||||
InitInternal();
|
||||
}
|
||||
|
||||
void Init(const std::set<I> &input) {
|
||||
CopySetToVector(input, &slow_set_);
|
||||
InitInternal();
|
||||
}
|
||||
|
||||
explicit ConstIntegerSet(const std::vector<I> &input): slow_set_(input) {
|
||||
SortAndUniq(&slow_set_);
|
||||
InitInternal();
|
||||
}
|
||||
explicit ConstIntegerSet(const std::set<I> &input) {
|
||||
CopySetToVector(input, &slow_set_);
|
||||
InitInternal();
|
||||
}
|
||||
explicit ConstIntegerSet(const ConstIntegerSet<I> &other):
|
||||
slow_set_(other.slow_set_) {
|
||||
InitInternal();
|
||||
}
|
||||
|
||||
int count(I i) const; // returns 1 or 0.
|
||||
|
||||
typedef typename std::vector<I>::const_iterator iterator;
|
||||
iterator begin() const { return slow_set_.begin(); }
|
||||
iterator end() const { return slow_set_.end(); }
|
||||
size_t size() const { return slow_set_.size(); }
|
||||
bool empty() const { return slow_set_.empty(); }
|
||||
|
||||
void Write(std::ostream &os, bool binary) const;
|
||||
void Read(std::istream &is, bool binary);
|
||||
|
||||
private:
|
||||
I lowest_member_;
|
||||
I highest_member_;
|
||||
bool contiguous_;
|
||||
bool quick_;
|
||||
std::vector<bool> quick_set_;
|
||||
std::vector<I> slow_set_;
|
||||
void InitInternal();
|
||||
};
|
||||
|
||||
} // end namespace kaldi
|
||||
|
||||
#include "util/const-integer-set-inl.h"
|
||||
|
||||
#endif // KALDI_UTIL_CONST_INTEGER_SET_H_
|
||||
@@ -0,0 +1,200 @@
|
||||
// util/edit-distance-inl.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation; Haihua Xu; Yanmin Qian
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_UTIL_EDIT_DISTANCE_INL_H_
|
||||
#define KALDI_UTIL_EDIT_DISTANCE_INL_H_
|
||||
#include <algorithm>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "util/stl-utils.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
template<class T>
|
||||
int32 LevenshteinEditDistance(const std::vector<T> &a,
|
||||
const std::vector<T> &b) {
|
||||
// Algorithm:
|
||||
// write A and B for the sequences, with elements a_0 ..
|
||||
// let |A| = M and |B| = N be the lengths, and have
|
||||
// elements a_0 ... a_{M-1} and b_0 ... b_{N-1}.
|
||||
// We are computing the recursion
|
||||
// E(m, n) = min( E(m-1, n-1) + (1-delta(a_{m-1}, b_{n-1})),
|
||||
// E(m-1, n) + 1,
|
||||
// E(m, n-1) + 1).
|
||||
// where E(m, n) is defined for m = 0..M and n = 0..N and out-of-
|
||||
// bounds quantities are considered to be infinity (i.e. the
|
||||
// recursion does not visit them).
|
||||
|
||||
// We do this computation using a vector e of size N+1.
|
||||
// The outer iterations range over m = 0..M.
|
||||
|
||||
int M = a.size(), N = b.size();
|
||||
std::vector<int32> e(N+1);
|
||||
std::vector<int32> e_tmp(N+1);
|
||||
// initialize e.
|
||||
for (size_t i = 0; i < e.size(); i++)
|
||||
e[i] = i;
|
||||
for (int32 m = 1; m <= M; m++) {
|
||||
// computing E(m, .) from E(m-1, .)
|
||||
// handle special case n = 0:
|
||||
e_tmp[0] = e[0] + 1;
|
||||
|
||||
for (int32 n = 1; n <= N; n++) {
|
||||
int32 term1 = e[n-1] + (a[m-1] == b[n-1] ? 0 : 1);
|
||||
int32 term2 = e[n] + 1;
|
||||
int32 term3 = e_tmp[n-1] + 1;
|
||||
e_tmp[n] = std::min(term1, std::min(term2, term3));
|
||||
}
|
||||
e = e_tmp;
|
||||
}
|
||||
return e.back();
|
||||
}
|
||||
//
|
||||
struct error_stats {
|
||||
int32 ins_num;
|
||||
int32 del_num;
|
||||
int32 sub_num;
|
||||
int32 total_cost; // minimum total cost to the current alignment.
|
||||
};
|
||||
// Note that both hyp and ref should not contain noise word in
|
||||
// the following implementation.
|
||||
|
||||
template<class T>
|
||||
int32 LevenshteinEditDistance(const std::vector<T> &ref,
|
||||
const std::vector<T> &hyp,
|
||||
int32 *ins, int32 *del, int32 *sub) {
|
||||
// temp sequence to remember error type and stats.
|
||||
std::vector<error_stats> e(ref.size()+1);
|
||||
std::vector<error_stats> cur_e(ref.size()+1);
|
||||
// initialize the first hypothesis aligned to the reference at each
|
||||
// position:[hyp_index =0][ref_index]
|
||||
for (size_t i =0; i < e.size(); i ++) {
|
||||
e[i].ins_num = 0;
|
||||
e[i].sub_num = 0;
|
||||
e[i].del_num = i;
|
||||
e[i].total_cost = i;
|
||||
}
|
||||
|
||||
// for other alignments
|
||||
for (size_t hyp_index = 1; hyp_index <= hyp.size(); hyp_index ++) {
|
||||
cur_e[0] = e[0];
|
||||
cur_e[0].ins_num++;
|
||||
cur_e[0].total_cost++;
|
||||
for (size_t ref_index = 1; ref_index <= ref.size(); ref_index ++) {
|
||||
int32 ins_err = e[ref_index].total_cost + 1;
|
||||
int32 del_err = cur_e[ref_index-1].total_cost + 1;
|
||||
int32 sub_err = e[ref_index-1].total_cost;
|
||||
if (hyp[hyp_index-1] != ref[ref_index-1])
|
||||
sub_err++;
|
||||
|
||||
if (sub_err < ins_err && sub_err < del_err) {
|
||||
cur_e[ref_index] =e[ref_index-1];
|
||||
if (hyp[hyp_index-1] != ref[ref_index-1])
|
||||
cur_e[ref_index].sub_num++; // substitution error should be increased
|
||||
cur_e[ref_index].total_cost = sub_err;
|
||||
} else if (del_err < ins_err) {
|
||||
cur_e[ref_index] = cur_e[ref_index-1];
|
||||
cur_e[ref_index].total_cost = del_err;
|
||||
cur_e[ref_index].del_num++; // deletion number is increased.
|
||||
} else {
|
||||
cur_e[ref_index] = e[ref_index];
|
||||
cur_e[ref_index].total_cost = ins_err;
|
||||
cur_e[ref_index].ins_num++; // insertion number is increased.
|
||||
}
|
||||
}
|
||||
e = cur_e; // alternate for the next recursion.
|
||||
}
|
||||
size_t ref_index = e.size()-1;
|
||||
*ins = e[ref_index].ins_num, *del =
|
||||
e[ref_index].del_num, *sub = e[ref_index].sub_num;
|
||||
return e[ref_index].total_cost;
|
||||
}
|
||||
|
||||
template<class T>
|
||||
int32 LevenshteinAlignment(const std::vector<T> &a,
|
||||
const std::vector<T> &b,
|
||||
T eps_symbol,
|
||||
std::vector<std::pair<T, T> > *output) {
|
||||
// Check inputs:
|
||||
{
|
||||
KALDI_ASSERT(output != NULL);
|
||||
for (size_t i = 0; i < a.size(); i++) KALDI_ASSERT(a[i] != eps_symbol);
|
||||
for (size_t i = 0; i < b.size(); i++) KALDI_ASSERT(b[i] != eps_symbol);
|
||||
}
|
||||
output->clear();
|
||||
// This is very memory-inefficiently implemented using a vector of vectors.
|
||||
size_t M = a.size(), N = b.size();
|
||||
size_t m, n;
|
||||
std::vector<std::vector<int32> > e(M+1);
|
||||
for (m = 0; m <=M; m++) e[m].resize(N+1);
|
||||
for (n = 0; n <= N; n++)
|
||||
e[0][n] = n;
|
||||
for (m = 1; m <= M; m++) {
|
||||
e[m][0] = e[m-1][0] + 1;
|
||||
for (n = 1; n <= N; n++) {
|
||||
int32 sub_or_ok = e[m-1][n-1] + (a[m-1] == b[n-1] ? 0 : 1);
|
||||
int32 del = e[m-1][n] + 1; // assumes a == ref, b == hyp.
|
||||
int32 ins = e[m][n-1] + 1;
|
||||
e[m][n] = std::min(sub_or_ok, std::min(del, ins));
|
||||
}
|
||||
}
|
||||
// get time-reversed output first: trace back.
|
||||
m = M;
|
||||
n = N;
|
||||
while (m != 0 || n != 0) {
|
||||
size_t last_m, last_n;
|
||||
if (m == 0) {
|
||||
last_m = m;
|
||||
last_n = n-1;
|
||||
} else if (n == 0) {
|
||||
last_m = m-1;
|
||||
last_n = n;
|
||||
} else {
|
||||
int32 sub_or_ok = e[m-1][n-1] + (a[m-1] == b[n-1] ? 0 : 1);
|
||||
int32 del = e[m-1][n] + 1; // assumes a == ref, b == hyp.
|
||||
int32 ins = e[m][n-1] + 1;
|
||||
// choose sub_or_ok if all else equal.
|
||||
if (sub_or_ok <= std::min(del, ins)) {
|
||||
last_m = m-1;
|
||||
last_n = n-1;
|
||||
} else {
|
||||
if (del <= ins) { // choose del over ins if equal.
|
||||
last_m = m-1;
|
||||
last_n = n;
|
||||
} else {
|
||||
last_m = m;
|
||||
last_n = n-1;
|
||||
}
|
||||
}
|
||||
}
|
||||
T a_sym, b_sym;
|
||||
a_sym = (last_m == m ? eps_symbol : a[last_m]);
|
||||
b_sym = (last_n == n ? eps_symbol : b[last_n]);
|
||||
output->push_back(std::make_pair(a_sym, b_sym));
|
||||
m = last_m;
|
||||
n = last_n;
|
||||
}
|
||||
ReverseVector(output);
|
||||
return e[M][N];
|
||||
}
|
||||
|
||||
|
||||
} // end namespace kaldi
|
||||
|
||||
#endif // KALDI_UTIL_EDIT_DISTANCE_INL_H_
|
||||
@@ -0,0 +1,267 @@
|
||||
// util/edit-distance-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation; Haihua Xu
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#include "base/kaldi-common.h"
|
||||
#include "util/edit-distance.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
void TestEditDistance() {
|
||||
std::vector<int32> a;
|
||||
std::vector<int32> b;
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 0);
|
||||
|
||||
a.push_back(1);
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 1);
|
||||
|
||||
b.push_back(1);
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 0);
|
||||
|
||||
b.push_back(2);
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 1);
|
||||
|
||||
a.push_back(2);
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 0);
|
||||
|
||||
a.push_back(3);
|
||||
a.push_back(4);
|
||||
b.push_back(4);
|
||||
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 1);
|
||||
|
||||
a.push_back(5);
|
||||
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 2);
|
||||
|
||||
b.push_back(6);
|
||||
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 2);
|
||||
|
||||
a.push_back(1);
|
||||
b.push_back(1);
|
||||
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 2);
|
||||
|
||||
b.push_back(10);
|
||||
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 3);
|
||||
}
|
||||
|
||||
|
||||
void TestEditDistanceString() {
|
||||
std::vector<std::string> a;
|
||||
std::vector<std::string> b;
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 0);
|
||||
|
||||
a.push_back("1");
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 1);
|
||||
|
||||
b.push_back("1");
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 0);
|
||||
|
||||
b.push_back("2");
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 1);
|
||||
|
||||
a.push_back("2");
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 0);
|
||||
|
||||
a.push_back("3");
|
||||
a.push_back("4");
|
||||
b.push_back("4");
|
||||
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 1);
|
||||
|
||||
a.push_back("5");
|
||||
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 2);
|
||||
|
||||
b.push_back("6");
|
||||
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 2);
|
||||
|
||||
a.push_back("1");
|
||||
b.push_back("1");
|
||||
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 2);
|
||||
|
||||
b.push_back("10");
|
||||
|
||||
KALDI_ASSERT(LevenshteinEditDistance(a, b) == 3);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// edit distance calculate
|
||||
void TestEditDistance2() {
|
||||
std::vector<int32> hyp;
|
||||
std::vector<int32> ref;
|
||||
int32 ins, del, sub, total_cost;
|
||||
// initialize hypothesis
|
||||
hyp.push_back(1);
|
||||
hyp.push_back(3);
|
||||
hyp.push_back(4);
|
||||
hyp.push_back(5);
|
||||
// initialize reference
|
||||
ref.push_back(2);
|
||||
ref.push_back(3);
|
||||
ref.push_back(4);
|
||||
ref.push_back(5);
|
||||
ref.push_back(6);
|
||||
ref.push_back(7);
|
||||
total_cost = LevenshteinEditDistance(ref, hyp, &ins, &del, &sub);
|
||||
KALDI_ASSERT(total_cost == 3 && ins == 0 && del == 2 && sub == 1);
|
||||
|
||||
std::swap(hyp, ref);
|
||||
total_cost = LevenshteinEditDistance(ref, hyp, &ins, &del, &sub);
|
||||
KALDI_ASSERT(total_cost == 3 && ins == 2 && del == 0 && sub == 1);
|
||||
|
||||
hyp.clear();
|
||||
ref.clear();
|
||||
hyp.push_back(1);
|
||||
ref.push_back(1);
|
||||
total_cost = LevenshteinEditDistance(ref, hyp, &ins, &del, &sub);
|
||||
KALDI_ASSERT(total_cost == 0 && ins+del+sub == 0);
|
||||
hyp.push_back(2);
|
||||
ref.push_back(3);
|
||||
total_cost = LevenshteinEditDistance(ref, hyp, &ins, &del, &sub);
|
||||
KALDI_ASSERT(total_cost == 1 && ins == 0 && del == 0 && sub == 1);
|
||||
// randomized test
|
||||
size_t num = 0;
|
||||
for (; num < 1000; num ++) {
|
||||
int32 hyp_len = Rand()%11;
|
||||
int32 ref_len = Rand()%3;
|
||||
hyp.resize(hyp_len);
|
||||
ref.resize(ref_len);
|
||||
|
||||
int32 index = 0;
|
||||
for (; index < hyp_len; index ++)
|
||||
hyp[index] = Rand()%4;
|
||||
for (index = 0; index < ref_len; index ++)
|
||||
ref[index] = Rand()%4;
|
||||
// current version
|
||||
total_cost = LevenshteinEditDistance(ref, hyp, &ins, &del, &sub);
|
||||
// previous version
|
||||
int32 total_cost2 = LevenshteinEditDistance(hyp, ref);
|
||||
// verify both are the same
|
||||
KALDI_ASSERT(total_cost == total_cost2);
|
||||
KALDI_ASSERT(ins+del+sub == total_cost);
|
||||
KALDI_ASSERT(del-ins == static_cast<int32>(ref.size() -hyp.size()));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
// edit distance calculate
|
||||
void TestEditDistance2String() {
|
||||
std::vector<std::string> hyp;
|
||||
std::vector<std::string> ref;
|
||||
int32 ins, del, sub, total_cost;
|
||||
// initialize hypothesis
|
||||
hyp.push_back("1");
|
||||
hyp.push_back("3");
|
||||
hyp.push_back("4");
|
||||
hyp.push_back("5");
|
||||
// initialize reference
|
||||
ref.push_back("2");
|
||||
ref.push_back("3");
|
||||
ref.push_back("4");
|
||||
ref.push_back("5");
|
||||
ref.push_back("6");
|
||||
ref.push_back("7");
|
||||
total_cost = LevenshteinEditDistance(ref, hyp, &ins, &del, &sub);
|
||||
KALDI_ASSERT(total_cost == 3 && ins == 0 && del == 2 && sub == 1);
|
||||
|
||||
std::swap(hyp, ref);
|
||||
total_cost = LevenshteinEditDistance(ref, hyp, &ins, &del, &sub);
|
||||
KALDI_ASSERT(total_cost == 3 && ins == 2 && del == 0 && sub == 1);
|
||||
|
||||
hyp.clear();
|
||||
ref.clear();
|
||||
hyp.push_back("1");
|
||||
ref.push_back("1");
|
||||
total_cost = LevenshteinEditDistance(ref, hyp, &ins, &del, &sub);
|
||||
KALDI_ASSERT(total_cost == 0 && ins+del+sub == 0);
|
||||
hyp.push_back("2");
|
||||
ref.push_back("3");
|
||||
total_cost = LevenshteinEditDistance(ref, hyp, &ins, &del, &sub);
|
||||
KALDI_ASSERT(total_cost == 1 && ins == 0 && del == 0 && sub == 1);
|
||||
// randomized test
|
||||
size_t num = 0;
|
||||
for (; num < 1000; num ++) {
|
||||
int32 hyp_len = Rand()%11;
|
||||
int32 ref_len = Rand()%3;
|
||||
hyp.resize(hyp_len);
|
||||
ref.resize(ref_len);
|
||||
|
||||
int32 index = 0;
|
||||
for (; index < hyp_len; index ++)
|
||||
hyp[index] = Rand()%4;
|
||||
for (index = 0; index < ref_len; index ++)
|
||||
ref[index] = Rand()%4;
|
||||
// current version
|
||||
total_cost = LevenshteinEditDistance(ref, hyp, &ins, &del, &sub);
|
||||
// previous version
|
||||
int32 total_cost2 = LevenshteinEditDistance(hyp, ref);
|
||||
// verify both are the same
|
||||
KALDI_ASSERT(total_cost == total_cost2);
|
||||
KALDI_ASSERT(ins+del+sub == total_cost);
|
||||
KALDI_ASSERT(del-ins == static_cast<int32>(ref.size() -hyp.size()));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
void TestLevenshteinAlignment() {
|
||||
for (size_t i = 0; i < 100; i++) {
|
||||
size_t a_sz = Rand() % 5, b_sz = Rand() % 5;
|
||||
std::vector<int32> a, b;
|
||||
for (size_t j = 0; j < a_sz; j++) a.push_back(Rand() % 10);
|
||||
for (size_t j = 0; j < b_sz; j++) b.push_back(Rand() % 10);
|
||||
int32 eps_sym = -1;
|
||||
std::vector<std::pair<int32, int32> > ans;
|
||||
|
||||
int32 e1 = LevenshteinEditDistance(a, b),
|
||||
e2 = LevenshteinAlignment(a, b, eps_sym, &ans);
|
||||
KALDI_ASSERT(e1 == e2);
|
||||
|
||||
std::vector<int32> a2, b2;
|
||||
for (size_t i = 0; i < ans.size(); i++) {
|
||||
if (ans[i].first != eps_sym) a2.push_back(ans[i].first);
|
||||
if (ans[i].second != eps_sym) b2.push_back(ans[i].second);
|
||||
}
|
||||
KALDI_ASSERT(a == a2);
|
||||
KALDI_ASSERT(b == b2);
|
||||
}
|
||||
}
|
||||
|
||||
} // end namespace kaldi
|
||||
|
||||
int main() {
|
||||
using namespace kaldi;
|
||||
TestEditDistance();
|
||||
TestEditDistanceString();
|
||||
TestEditDistance2();
|
||||
TestEditDistance2String();
|
||||
TestLevenshteinAlignment();
|
||||
std::cout << "Test OK\n";
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
// util/edit-distance.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation; Haihua Xu
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#ifndef KALDI_UTIL_EDIT_DISTANCE_H_
|
||||
#define KALDI_UTIL_EDIT_DISTANCE_H_
|
||||
#include <vector>
|
||||
#include <set>
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <cassert>
|
||||
#include <utility>
|
||||
#include "util/edit-distance-inl.h"
|
||||
#include "base/kaldi-types.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
// Compute the edit-distance between two strings.
|
||||
template<class T>
|
||||
int32 LevenshteinEditDistance(const std::vector<T> &a,
|
||||
const std::vector<T> &b);
|
||||
|
||||
|
||||
// edit distance calculation with conventional method.
|
||||
// note: noise word must be filtered out from the hypothesis and
|
||||
// reference sequence
|
||||
// before the following procedure conducted.
|
||||
template<class T>
|
||||
int32 LevenshteinEditDistance(const std::vector<T> &ref,
|
||||
const std::vector<T> &hyp,
|
||||
int32 *ins, int32 *del, int32 *sub);
|
||||
|
||||
// This version of the edit-distance computation outputs the alignment
|
||||
// between the two. This is a vector of pairs of (symbol a, symbol b).
|
||||
// The epsilon symbol (eps_symbol) must not occur in sequences a or b.
|
||||
// Where one aligned to no symbol in the other (insertion or deletion),
|
||||
// epsilon will be the corresponding member of the pair.
|
||||
// It returns the edit-distance between the two strings.
|
||||
|
||||
template<class T>
|
||||
int32 LevenshteinAlignment(const std::vector<T> &a,
|
||||
const std::vector<T> &b,
|
||||
T eps_symbol,
|
||||
std::vector<std::pair<T, T> > *output);
|
||||
|
||||
} // end namespace kaldi
|
||||
|
||||
#endif // KALDI_UTIL_EDIT_DISTANCE_H_
|
||||
@@ -0,0 +1,194 @@
|
||||
// util/hash-list-inl.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2013 Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#ifndef KALDI_UTIL_HASH_LIST_INL_H_
|
||||
#define KALDI_UTIL_HASH_LIST_INL_H_
|
||||
|
||||
// Do not include this file directly. It is included by fast-hash.h
|
||||
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
template<class I, class T> HashList<I, T>::HashList() {
|
||||
list_head_ = NULL;
|
||||
bucket_list_tail_ = static_cast<size_t>(-1); // invalid.
|
||||
hash_size_ = 0;
|
||||
freed_head_ = NULL;
|
||||
}
|
||||
|
||||
template<class I, class T> void HashList<I, T>::SetSize(size_t size) {
|
||||
hash_size_ = size;
|
||||
KALDI_ASSERT(list_head_ == NULL &&
|
||||
bucket_list_tail_ == static_cast<size_t>(-1)); // make sure empty.
|
||||
if (size > buckets_.size())
|
||||
buckets_.resize(size, HashBucket(0, NULL));
|
||||
}
|
||||
|
||||
template<class I, class T>
|
||||
typename HashList<I, T>::Elem* HashList<I, T>::Clear() {
|
||||
// Clears the hashtable and gives ownership of the currently contained list
|
||||
// to the user.
|
||||
for (size_t cur_bucket = bucket_list_tail_;
|
||||
cur_bucket != static_cast<size_t>(-1);
|
||||
cur_bucket = buckets_[cur_bucket].prev_bucket) {
|
||||
buckets_[cur_bucket].last_elem = NULL; // this is how we indicate "empty".
|
||||
}
|
||||
bucket_list_tail_ = static_cast<size_t>(-1);
|
||||
Elem *ans = list_head_;
|
||||
list_head_ = NULL;
|
||||
return ans;
|
||||
}
|
||||
|
||||
template<class I, class T>
|
||||
const typename HashList<I, T>::Elem* HashList<I, T>::GetList() const {
|
||||
return list_head_;
|
||||
}
|
||||
|
||||
template<class I, class T>
|
||||
inline void HashList<I, T>::Delete(Elem *e) {
|
||||
e->tail = freed_head_;
|
||||
freed_head_ = e;
|
||||
}
|
||||
|
||||
template<class I, class T>
|
||||
inline typename HashList<I, T>::Elem* HashList<I, T>::Find(I key) {
|
||||
size_t index = (static_cast<size_t>(key) % hash_size_);
|
||||
HashBucket &bucket = buckets_[index];
|
||||
if (bucket.last_elem == NULL) {
|
||||
return NULL; // empty bucket.
|
||||
} else {
|
||||
Elem *head = (bucket.prev_bucket == static_cast<size_t>(-1) ?
|
||||
list_head_ :
|
||||
buckets_[bucket.prev_bucket].last_elem->tail),
|
||||
*tail = bucket.last_elem->tail;
|
||||
for (Elem *e = head; e != tail; e = e->tail)
|
||||
if (e->key == key) return e;
|
||||
return NULL; // Not found.
|
||||
}
|
||||
}
|
||||
|
||||
template<class I, class T>
|
||||
inline typename HashList<I, T>::Elem* HashList<I, T>::New() {
|
||||
if (freed_head_) {
|
||||
Elem *ans = freed_head_;
|
||||
freed_head_ = freed_head_->tail;
|
||||
return ans;
|
||||
} else {
|
||||
Elem *tmp = new Elem[allocate_block_size_];
|
||||
for (size_t i = 0; i+1 < allocate_block_size_; i++)
|
||||
tmp[i].tail = tmp+i+1;
|
||||
tmp[allocate_block_size_-1].tail = NULL;
|
||||
freed_head_ = tmp;
|
||||
allocated_.push_back(tmp);
|
||||
return this->New();
|
||||
}
|
||||
}
|
||||
|
||||
template<class I, class T>
|
||||
HashList<I, T>::~HashList() {
|
||||
// First test whether we had any memory leak within the
|
||||
// HashList, i.e. things for which the user did not call Delete().
|
||||
size_t num_in_list = 0, num_allocated = 0;
|
||||
for (Elem *e = freed_head_; e != NULL; e = e->tail)
|
||||
num_in_list++;
|
||||
for (size_t i = 0; i < allocated_.size(); i++) {
|
||||
num_allocated += allocate_block_size_;
|
||||
delete[] allocated_[i];
|
||||
}
|
||||
if (num_in_list != num_allocated) {
|
||||
KALDI_WARN << "Possible memory leak: " << num_in_list
|
||||
<< " != " << num_allocated
|
||||
<< ": you might have forgotten to call Delete on "
|
||||
<< "some Elems";
|
||||
}
|
||||
}
|
||||
|
||||
template<class I, class T>
|
||||
inline typename HashList<I, T>::Elem* HashList<I, T>::Insert(I key, T val) {
|
||||
size_t index = (static_cast<size_t>(key) % hash_size_);
|
||||
HashBucket &bucket = buckets_[index];
|
||||
// Check the element is existing or not.
|
||||
if (bucket.last_elem != NULL) {
|
||||
Elem *head = (bucket.prev_bucket == static_cast<size_t>(-1) ?
|
||||
list_head_ :
|
||||
buckets_[bucket.prev_bucket].last_elem->tail),
|
||||
*tail = bucket.last_elem->tail;
|
||||
for (Elem *e = head; e != tail; e = e->tail)
|
||||
if (e->key == key) return e;
|
||||
}
|
||||
|
||||
// This is a new element. Insert it.
|
||||
Elem *elem = New();
|
||||
elem->key = key;
|
||||
elem->val = val;
|
||||
if (bucket.last_elem == NULL) { // Unoccupied bucket. Insert at
|
||||
// head of bucket list (which is tail of regular list, they go in
|
||||
// opposite directions).
|
||||
if (bucket_list_tail_ == static_cast<size_t>(-1)) {
|
||||
// list was empty so this is the first elem.
|
||||
KALDI_ASSERT(list_head_ == NULL);
|
||||
list_head_ = elem;
|
||||
} else {
|
||||
// link in to the chain of Elems
|
||||
buckets_[bucket_list_tail_].last_elem->tail = elem;
|
||||
}
|
||||
elem->tail = NULL;
|
||||
bucket.last_elem = elem;
|
||||
bucket.prev_bucket = bucket_list_tail_;
|
||||
bucket_list_tail_ = index;
|
||||
} else {
|
||||
// Already-occupied bucket. Insert at tail of list of elements within
|
||||
// the bucket.
|
||||
elem->tail = bucket.last_elem->tail;
|
||||
bucket.last_elem->tail = elem;
|
||||
bucket.last_elem = elem;
|
||||
}
|
||||
return elem;
|
||||
}
|
||||
|
||||
template<class I, class T>
|
||||
void HashList<I, T>::InsertMore(I key, T val) {
|
||||
size_t index = (static_cast<size_t>(key) % hash_size_);
|
||||
HashBucket &bucket = buckets_[index];
|
||||
Elem *elem = New();
|
||||
elem->key = key;
|
||||
elem->val = val;
|
||||
|
||||
KALDI_ASSERT(bucket.last_elem != NULL); // assume one element is already here
|
||||
if (bucket.last_elem->key == key) { // standard behavior: add as last element
|
||||
elem->tail = bucket.last_elem->tail;
|
||||
bucket.last_elem->tail = elem;
|
||||
bucket.last_elem = elem;
|
||||
return;
|
||||
}
|
||||
Elem *e = (bucket.prev_bucket == static_cast<size_t>(-1) ?
|
||||
list_head_ : buckets_[bucket.prev_bucket].last_elem->tail);
|
||||
// find place to insert in linked list
|
||||
while (e != bucket.last_elem->tail && e->key != key) e = e->tail;
|
||||
KALDI_ASSERT(e->key == key); // not found? - should not happen
|
||||
elem->tail = e->tail;
|
||||
e->tail = elem;
|
||||
}
|
||||
|
||||
|
||||
} // end namespace kaldi
|
||||
|
||||
#endif // KALDI_UTIL_HASH_LIST_INL_H_
|
||||
@@ -0,0 +1,107 @@
|
||||
// util/hash-list-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2013 Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#include "util/hash-list.h"
|
||||
#include <map> // for baseline.
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
template<class Int, class T> void TestHashList() {
|
||||
typedef typename HashList<Int, T>::Elem Elem;
|
||||
|
||||
HashList<Int, T> hash;
|
||||
hash.SetSize(200); // must be called before use.
|
||||
std::map<Int, T> m1;
|
||||
for (size_t j = 0; j < 50; j++) {
|
||||
Int key = Rand() % 200;
|
||||
T val = Rand() % 50;
|
||||
m1[key] = val;
|
||||
Elem *e = hash.Find(key);
|
||||
if (e) e->val = val;
|
||||
else hash.Insert(key, val);
|
||||
}
|
||||
|
||||
|
||||
std::map<Int, T> m2;
|
||||
|
||||
for (int i = 0; i < 100; i++) {
|
||||
m2.clear();
|
||||
for (typename std::map<Int, T>::const_iterator iter = m1.begin();
|
||||
iter != m1.end();
|
||||
iter++) {
|
||||
m2[iter->first + 1] = iter->second;
|
||||
}
|
||||
std::swap(m1, m2);
|
||||
|
||||
Elem *h = hash.Clear(), *tmp;
|
||||
|
||||
hash.SetSize(100 + Rand() % 100); // note, SetSize is relatively cheap
|
||||
// operation as long as we are not increasing the size more than it's ever
|
||||
// previously been increased to.
|
||||
|
||||
for (; h != NULL; h = tmp) {
|
||||
hash.Insert(h->key + 1, h->val);
|
||||
tmp = h->tail;
|
||||
hash.Delete(h); // think of this like calling delete.
|
||||
}
|
||||
|
||||
// Now make sure h and m2 are the same.
|
||||
const Elem *list = hash.GetList();
|
||||
size_t count = 0;
|
||||
for (; list != NULL; list = list->tail, count++) {
|
||||
KALDI_ASSERT(m1[list->key] == list->val);
|
||||
}
|
||||
|
||||
for (size_t j = 0; j < 10; j++) {
|
||||
Int key = Rand() % 200;
|
||||
bool found_m1 = (m1.find(key) != m1.end());
|
||||
if (found_m1) m1[key];
|
||||
Elem *e = hash.Find(key);
|
||||
KALDI_ASSERT((e != NULL) == found_m1);
|
||||
if (found_m1)
|
||||
KALDI_ASSERT(m1[key] == e->val);
|
||||
}
|
||||
|
||||
KALDI_ASSERT(m1.size() == count);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
} // end namespace kaldi
|
||||
|
||||
|
||||
|
||||
int main() {
|
||||
using namespace kaldi;
|
||||
for (size_t i = 0;i < 3;i++) {
|
||||
TestHashList<int, unsigned int>();
|
||||
TestHashList<unsigned int, int>();
|
||||
TestHashList<int16, int32>();
|
||||
TestHashList<int16, int32>();
|
||||
TestHashList<char, unsigned char>();
|
||||
TestHashList<unsigned char, int>();
|
||||
}
|
||||
std::cout << "Test OK.\n";
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
// util/hash-list.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2013 Johns Hopkins University (author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#ifndef KALDI_UTIL_HASH_LIST_H_
|
||||
#define KALDI_UTIL_HASH_LIST_H_
|
||||
#include <vector>
|
||||
#include <set>
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <cassert>
|
||||
#include "util/stl-utils.h"
|
||||
|
||||
|
||||
/* This header provides utilities for a structure that's used in a decoder (but
|
||||
is quite generic in nature so we implement and test it separately).
|
||||
Basically it's a singly-linked list, but implemented in such a way that we
|
||||
can quickly search for elements in the list. We give it a slightly richer
|
||||
interface than just a hash and a list. The idea is that we want to separate
|
||||
the hash part and the list part: basically, in the decoder, we want to have a
|
||||
single hash for the current frame and the next frame, because by the time we
|
||||
need to access the hash for the next frame we no longer need the hash for the
|
||||
previous frame. So we have an operation that clears the hash but leaves the
|
||||
list structure intact. We also control memory management inside this object,
|
||||
to avoid repeated new's/deletes.
|
||||
|
||||
See hash-list-test.cc for an example of how to use this object.
|
||||
*/
|
||||
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
template<class I, class T> class HashList {
|
||||
public:
|
||||
struct Elem {
|
||||
I key;
|
||||
T val;
|
||||
Elem *tail;
|
||||
};
|
||||
|
||||
/// Constructor takes no arguments.
|
||||
/// Call SetSize to inform it of the likely size.
|
||||
HashList();
|
||||
|
||||
/// Clears the hash and gives the head of the current list to the user;
|
||||
/// ownership is transferred to the user (the user must call Delete()
|
||||
/// for each element in the list, at his/her leisure).
|
||||
Elem *Clear();
|
||||
|
||||
/// Gives the head of the current list to the user. Ownership retained in the
|
||||
/// class. Caution: in December 2013 the return type was changed to const
|
||||
/// Elem* and this function was made const. You may need to change some types
|
||||
/// of local Elem* variables to const if this produces compilation errors.
|
||||
const Elem *GetList() const;
|
||||
|
||||
/// Think of this like delete(). It is to be called for each Elem in turn
|
||||
/// after you "obtained ownership" by doing Clear(). This is not the opposite
|
||||
/// of. Insert, it is the opposite of New. It's really a memory operation.
|
||||
inline void Delete(Elem *e);
|
||||
|
||||
/// This should probably not be needed to be called directly by the user.
|
||||
/// Think of it as opposite
|
||||
/// to Delete();
|
||||
inline Elem *New();
|
||||
|
||||
/// Find tries to find this element in the current list using the hashtable.
|
||||
/// It returns NULL if not present. The Elem it returns is not owned by the
|
||||
/// user, it is part of the internal list owned by this object, but the user
|
||||
/// is free to modify the "val" element.
|
||||
inline Elem *Find(I key);
|
||||
|
||||
/// Insert inserts a new element into the hashtable/stored list.
|
||||
/// Because element keys in a hashtable are unique, this operation checks
|
||||
/// whether each inserted element has a key equivalent to the one of an
|
||||
/// element already in the hashtable. If so, the element is not inserted,
|
||||
/// returning an pointer to this existing element.
|
||||
inline Elem *Insert(I key, T val);
|
||||
|
||||
/// Insert inserts another element with same key into the hashtable/
|
||||
/// stored list.
|
||||
/// By calling this, the user asserts that one element with that key is
|
||||
/// already present.
|
||||
/// We insert it that way, that all elements with the same key
|
||||
/// follow each other.
|
||||
/// Find() will return the first one of the elements with the same key.
|
||||
inline void InsertMore(I key, T val);
|
||||
|
||||
/// SetSize tells the object how many hash buckets to allocate (should
|
||||
/// typically be at least twice the number of objects we expect to go in the
|
||||
/// structure, for fastest performance). It must be called while the hash
|
||||
/// is empty (e.g. after Clear() or after initializing the object, but before
|
||||
/// adding anything to the hash.
|
||||
void SetSize(size_t sz);
|
||||
|
||||
/// Returns current number of hash buckets.
|
||||
inline size_t Size() { return hash_size_; }
|
||||
|
||||
~HashList();
|
||||
private:
|
||||
|
||||
struct HashBucket {
|
||||
size_t prev_bucket; // index to next bucket (-1 if list tail). Note:
|
||||
// list of buckets goes in opposite direction to list of Elems.
|
||||
Elem *last_elem; // pointer to last element in this bucket (NULL if empty)
|
||||
inline HashBucket(size_t i, Elem *e): prev_bucket(i), last_elem(e) {}
|
||||
};
|
||||
|
||||
Elem *list_head_; // head of currently stored list.
|
||||
size_t bucket_list_tail_; // tail of list of active hash buckets.
|
||||
|
||||
size_t hash_size_; // number of hash buckets.
|
||||
|
||||
std::vector<HashBucket> buckets_;
|
||||
|
||||
Elem *freed_head_; // head of list of currently freed elements. [ready for
|
||||
// allocation]
|
||||
|
||||
std::vector<Elem*> allocated_; // list of allocated blocks.
|
||||
|
||||
static const size_t allocate_block_size_ = 1024; // Number of Elements to
|
||||
// allocate in one block. Must be largish so storing allocated_ doesn't
|
||||
// become a problem.
|
||||
};
|
||||
|
||||
|
||||
} // end namespace kaldi
|
||||
|
||||
#include "util/hash-list-inl.h"
|
||||
|
||||
#endif // KALDI_UTIL_HASH_LIST_H_
|
||||
@@ -0,0 +1,129 @@
|
||||
// util/kaldi-cygwin-io-inl.h
|
||||
|
||||
// Copyright 2015 Smart Action Company LLC (author: Kirill Katsnelson)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#ifndef KALDI_UTIL_KALDI_CYGWIN_IO_INL_H_
|
||||
#define KALDI_UTIL_KALDI_CYGWIN_IO_INL_H_
|
||||
|
||||
#ifndef _MSC_VER
|
||||
#error This is a Windows-compatibility file. Something went wery wrong.
|
||||
#endif
|
||||
|
||||
#include <string>
|
||||
|
||||
// This file is included only into kaldi-io.cc, and only if
|
||||
// KALDI_CYGWIN_COMPAT is enabled.
|
||||
//
|
||||
// The routines map unix-ey paths passed to Windows programs from shell
|
||||
// scripts in egs. Since shell scripts run under cygwin, they use cygwin's
|
||||
// own mount table and a mapping to the file system. It is quite possible to
|
||||
// create quite an intricate mapping that only own cygwin API would be able
|
||||
// to untangle. Unfortunately, the API to map between filenames is not
|
||||
// available to non-cygwin programs. Running cygpath for every file operation
|
||||
// would as well be cumbersome. So this is only a simplistic path resolution,
|
||||
// assuming that the default cygwin prefix /cygdrive is used, and that all
|
||||
// resolved unix-style full paths end up prefixed with /cygdrive. This is
|
||||
// quite a sensible approach. We'll also try to map /dev/null and /tmp/**,
|
||||
// die on all other /dev/** and warn about all other rooted paths.
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
static bool prefixp(const std::string& pfx, const std::string& str) {
|
||||
return pfx.length() <= str.length() &&
|
||||
std::equal(pfx.begin(), pfx.end(), str.begin());
|
||||
}
|
||||
|
||||
static std::string cygprefix("/cygdrive/");
|
||||
|
||||
static std::string MapCygwinPathNoTmp(const std::string &filename) {
|
||||
// UNC(?), relative, native Windows and empty paths are ok already.
|
||||
if (prefixp("//", filename) || !prefixp("/", filename))
|
||||
return filename;
|
||||
|
||||
// /dev/...
|
||||
if (filename == "/dev/null")
|
||||
return "\\\\.\\nul";
|
||||
if (prefixp("/dev/", filename)) {
|
||||
KALDI_ERR << "Unable to resolve path '" << filename
|
||||
<< "' - only have /dev/null here.";
|
||||
return "\\\\.\\invalid";
|
||||
}
|
||||
|
||||
// /cygdrive/?[/....]
|
||||
int preflen = cygprefix.size();
|
||||
if (prefixp(cygprefix, filename)
|
||||
&& filename.size() >= preflen + 1 && isalpha(filename[preflen])
|
||||
&& (filename.size() == preflen + 1 || filename[preflen + 1] == '/')) {
|
||||
return std::string() + filename[preflen] + ':' +
|
||||
(filename.size() > preflen + 1 ? filename.substr(preflen + 1) : "/");
|
||||
}
|
||||
|
||||
KALDI_WARN << "Unable to resolve path '" << filename
|
||||
<< "' - cannot map unix prefix. "
|
||||
<< "Will go on, but breakage will likely ensue.";
|
||||
return filename;
|
||||
}
|
||||
|
||||
// extern for unit testing.
|
||||
std::string MapCygwinPath(const std::string &filename) {
|
||||
// /tmp[/....]
|
||||
if (filename != "/tmp" && !prefixp("/tmp/", filename)) {
|
||||
return MapCygwinPathNoTmp(filename);
|
||||
}
|
||||
char *tmpdir = std::getenv("TMP");
|
||||
if (tmpdir == nullptr)
|
||||
tmpdir = std::getenv("TEMP");
|
||||
if (tmpdir == nullptr) {
|
||||
KALDI_ERR << "Unable to resolve path '" << filename
|
||||
<< "' - unable to find temporary directory. Set TMP.";
|
||||
return filename;
|
||||
}
|
||||
// Map the value of tmpdir again, as cygwin environment actually may contain
|
||||
// unix-style paths.
|
||||
return MapCygwinPathNoTmp(std::string(tmpdir) + filename.substr(4));
|
||||
}
|
||||
|
||||
// A popen implementation that passes the command line through cygwin
|
||||
// bash.exe. This is necessary since some piped commands are cygwin links
|
||||
// (e. g. fgrep is a soft link to grep), and some are #!-files, such as
|
||||
// gunzip which is a shell script that invokes gzip, or kaldi's own run.pl
|
||||
// which is a perl script.
|
||||
//
|
||||
// _popen uses cmd.exe or whatever shell is specified via the COMSPEC
|
||||
// variable. Unfortunately, it adds a hardcoded " /c " to it, so we cannot
|
||||
// just substitute the environment variable COMSPEC to point to bash.exe.
|
||||
// Instead, quote the command and pass it to bash via its -c switch.
|
||||
static FILE *CygwinCompatPopen(const char* command, const char* mode) {
|
||||
// To speed up command launch marginally, optionally accept full path
|
||||
// to bash.exe. This will not work if the path contains spaces, but
|
||||
// no sane person would install cygwin into a space-ridden path.
|
||||
const char* bash_exe = std::getenv("BASH_EXE");
|
||||
std::string qcmd(bash_exe != nullptr ? bash_exe : "bash.exe");
|
||||
qcmd += " -c \"";
|
||||
for (; *command; ++command) {
|
||||
if (*command == '\"')
|
||||
qcmd += '\"';
|
||||
qcmd += *command;
|
||||
}
|
||||
qcmd += '\"';
|
||||
|
||||
return _popen(qcmd.c_str(), mode);
|
||||
}
|
||||
|
||||
} // namespace kaldi
|
||||
|
||||
#endif // KALDI_UTIL_KALDI_CYGWIN_IO_INL_H_
|
||||
@@ -0,0 +1,46 @@
|
||||
// util/kaldi-io-inl.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#ifndef KALDI_UTIL_KALDI_IO_INL_H_
|
||||
#define KALDI_UTIL_KALDI_IO_INL_H_
|
||||
|
||||
#include<string>
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
bool Input::Open(const std::string &rxfilename, bool *binary) {
|
||||
return OpenInternal(rxfilename, true, binary);
|
||||
}
|
||||
|
||||
bool Input::OpenTextMode(const std::string &rxfilename) {
|
||||
return OpenInternal(rxfilename, false, NULL);
|
||||
}
|
||||
|
||||
bool Input::IsOpen() {
|
||||
return impl_ != NULL;
|
||||
}
|
||||
|
||||
bool Output::IsOpen() {
|
||||
return impl_ != NULL;
|
||||
}
|
||||
|
||||
|
||||
} // end namespace kaldi.
|
||||
|
||||
|
||||
#endif // KALDI_UTIL_KALDI_IO_INL_H_
|
||||
@@ -0,0 +1,370 @@
|
||||
// util/kaldi-io-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#ifndef _MSC_VER
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#include "base/io-funcs.h"
|
||||
#include "util/kaldi-io.h"
|
||||
#include "base/kaldi-math.h"
|
||||
#include "base/kaldi-utils.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
|
||||
|
||||
void UnitTestClassifyRxfilename() {
|
||||
KALDI_ASSERT(ClassifyRxfilename("") == kStandardInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename(" ") == kNoInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename(" a ") == kNoInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename("a ") == kNoInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename("a") == kFileInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename("-") == kStandardInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename("b|") == kPipeInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename("|b") == kNoInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename("b c|") == kPipeInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename(" b c|") == kPipeInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename("a b c:123") == kOffsetFileInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename("a b c:3") == kOffsetFileInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename("a b c:") == kFileInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename("a b c/3") == kFileInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename("ark,s,cs:a b c") == kNoInput);
|
||||
KALDI_ASSERT(ClassifyRxfilename("scp:a b c") == kNoInput);
|
||||
|
||||
}
|
||||
|
||||
|
||||
void UnitTestClassifyWxfilename() {
|
||||
KALDI_ASSERT(ClassifyWxfilename("") == kStandardOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename(" ") == kNoOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename(" a ") == kNoOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("a ") == kNoOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("a") == kFileOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("-") == kStandardOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("b|") == kNoOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("|b") == kPipeOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("| b ") == kPipeOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("b c|") == kNoOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("a b c:123") == kNoOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("ark,s,cs:a b c") == kNoOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("scp:a b c") == kNoOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("a b c:3") == kNoOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("a b c:") == kFileOutput);
|
||||
KALDI_ASSERT(ClassifyWxfilename("a b c/3") == kFileOutput);
|
||||
}
|
||||
|
||||
void UnitTestIoNew(bool binary) {
|
||||
{
|
||||
const char *filename = "tmpf";
|
||||
|
||||
Output ko(filename, binary);
|
||||
std::ostream &outfile = ko.Stream();
|
||||
if (!binary) outfile << "\t";
|
||||
int64 i1 = Rand() % 10000;
|
||||
WriteBasicType(outfile, binary, i1);
|
||||
uint16 i2 = Rand() % 10000;
|
||||
WriteBasicType(outfile, binary, i2);
|
||||
if (!binary) outfile << "\t";
|
||||
char c = Rand();
|
||||
WriteBasicType(outfile, binary, c);
|
||||
if (!binary && Rand()%2 == 0) outfile << " \n";
|
||||
std::vector<int32> vec1;
|
||||
WriteIntegerVector(outfile, binary, vec1);
|
||||
if (!binary && Rand()%2 == 0) outfile << " \n";
|
||||
std::vector<uint16> vec2;
|
||||
for (size_t i = 0; i < 10; i++) vec2.push_back(Rand()%100 - 10);
|
||||
WriteIntegerVector(outfile, binary, vec2);
|
||||
if (!binary) outfile << " \n";
|
||||
std::vector<char> vec3;
|
||||
for (size_t i = 0; i < 10; i++) vec3.push_back(Rand()%100);
|
||||
WriteIntegerVector(outfile, binary, vec3);
|
||||
if (!binary && Rand()%2 == 0) outfile << " \n";
|
||||
const char *token1 = "Hi";
|
||||
WriteToken(outfile, binary, token1);
|
||||
if (!binary) outfile << " \n";
|
||||
std::string token2 = "There.";
|
||||
WriteToken(outfile, binary, token2);
|
||||
if (!binary && Rand()%2 == 0) outfile << " \n";
|
||||
std::string token3 = "You.";
|
||||
WriteToken(outfile, binary, token3);
|
||||
if (!binary && Rand()%2 == 0) outfile << " ";
|
||||
float f1 = RandUniform();
|
||||
WriteBasicType(outfile, binary, f1);
|
||||
if (!binary && Rand()%2 == 0) outfile << "\t";
|
||||
float f2 = RandUniform();
|
||||
WriteBasicType(outfile, binary, f2);
|
||||
double d1 = RandUniform();
|
||||
WriteBasicType(outfile, binary, d1);
|
||||
if (!binary && Rand()%2 == 0) outfile << "\t";
|
||||
double d2 = RandUniform();
|
||||
WriteBasicType(outfile, binary, d2);
|
||||
if (!binary && Rand()%2 == 0) outfile << "\t";
|
||||
ko.Close();
|
||||
|
||||
{
|
||||
bool binary_in;
|
||||
Input ki(filename, &binary_in);
|
||||
std::istream &infile = ki.Stream();
|
||||
int64 i1_in;
|
||||
ReadBasicType(infile, binary_in, &i1_in);
|
||||
KALDI_ASSERT(i1_in == i1);
|
||||
uint16 i2_in;
|
||||
ReadBasicType(infile, binary_in, &i2_in);
|
||||
KALDI_ASSERT(i2_in == i2);
|
||||
char c_in;
|
||||
ReadBasicType(infile, binary_in, &c_in);
|
||||
KALDI_ASSERT(c_in == c);
|
||||
std::vector<int32> vec1_in;
|
||||
ReadIntegerVector(infile, binary_in, &vec1_in);
|
||||
KALDI_ASSERT(vec1_in == vec1);
|
||||
std::vector<uint16> vec2_in;
|
||||
ReadIntegerVector(infile, binary_in, &vec2_in);
|
||||
KALDI_ASSERT(vec2_in == vec2);
|
||||
std::vector<char> vec3_in;
|
||||
ReadIntegerVector(infile, binary_in, &vec3_in);
|
||||
KALDI_ASSERT(vec3_in == vec3);
|
||||
std::string token1_in, token2_in;
|
||||
KALDI_ASSERT(Peek(infile, binary_in) == static_cast<int>(*token1));
|
||||
ReadToken(infile, binary_in, &token1_in);
|
||||
KALDI_ASSERT(token1_in == (std::string)token1);
|
||||
ReadToken(infile, binary_in, &token2_in);
|
||||
KALDI_ASSERT(token2_in == token2);
|
||||
if (Rand() % 2 == 0)
|
||||
ExpectToken(infile, binary_in, token3.c_str());
|
||||
else
|
||||
ExpectToken(infile, binary_in, token3);
|
||||
float f1_in; // same type.
|
||||
ReadBasicType(infile, binary_in, &f1_in);
|
||||
AssertEqual(f1_in, f1);
|
||||
double f2_in; // wrong type.
|
||||
ReadBasicType(infile, binary_in, &f2_in);
|
||||
AssertEqual(f2_in, f2);
|
||||
double d1_in; // same type.
|
||||
ReadBasicType(infile, binary_in, &d1_in);
|
||||
AssertEqual(d1_in, d1);
|
||||
float d2_in; // wrong type.
|
||||
ReadBasicType(infile, binary_in, &d2_in);
|
||||
AssertEqual(d2_in, d2);
|
||||
KALDI_ASSERT(Peek(infile, binary_in) == -1);
|
||||
}
|
||||
unlink(filename);
|
||||
}
|
||||
}
|
||||
|
||||
void UnitTestIoPipe(bool binary) {
|
||||
// This is as UnitTestIoNew except with different filenames.
|
||||
{
|
||||
#if defined(_MSC_VER) && !defined(KALDI_CYGWIN_COMPAT)
|
||||
// self-invocation on Windows that emulates cat(1)
|
||||
const char *filename_out = "|kaldi-io-test cat > tmpf.gz",
|
||||
*filename_in = "kaldi-io-test cat tmpf.gz|";
|
||||
#else
|
||||
const char *filename_out = "|gzip -c > tmpf.gz",
|
||||
*filename_in = "gunzip -c tmpf.gz |";
|
||||
#endif
|
||||
|
||||
Output ko(filename_out, binary);
|
||||
std::ostream &outfile = ko.Stream();
|
||||
if (!binary) outfile << "\t";
|
||||
int64 i1 = Rand() % 10000;
|
||||
WriteBasicType(outfile, binary, i1);
|
||||
uint16 i2 = Rand() % 10000;
|
||||
WriteBasicType(outfile, binary, i2);
|
||||
if (!binary) outfile << "\t";
|
||||
char c = Rand();
|
||||
WriteBasicType(outfile, binary, c);
|
||||
if (!binary && Rand()%2 == 0) outfile << " \n";
|
||||
std::vector<int32> vec1;
|
||||
WriteIntegerVector(outfile, binary, vec1);
|
||||
if (!binary && Rand()%2 == 0) outfile << " \n";
|
||||
std::vector<uint16> vec2;
|
||||
for (size_t i = 0; i < 10; i++) vec2.push_back(Rand()%100 - 10);
|
||||
WriteIntegerVector(outfile, binary, vec2);
|
||||
if (!binary) outfile << " \n";
|
||||
WriteToken(outfile, binary, "<foo>");
|
||||
std::vector<char> vec3;
|
||||
for (size_t i = 0; i < 10; i++) vec3.push_back(Rand()%100);
|
||||
WriteIntegerVector(outfile, binary, vec3);
|
||||
if (!binary && Rand()%2 == 0) outfile << " \n";
|
||||
const char *token1 = "Hi";
|
||||
WriteToken(outfile, binary, token1);
|
||||
if (!binary) outfile << " \n";
|
||||
std::string token2 = "There.";
|
||||
WriteToken(outfile, binary, token2);
|
||||
if (!binary && Rand()%2 == 0) outfile << " \n";
|
||||
std::string token3 = "You.";
|
||||
WriteToken(outfile, binary, token3);
|
||||
if (!binary && Rand()%2 == 0) outfile << " ";
|
||||
float f1 = RandUniform();
|
||||
WriteBasicType(outfile, binary, f1);
|
||||
if (!binary && Rand()%2 == 0) outfile << "\t";
|
||||
float f2 = RandUniform();
|
||||
WriteBasicType(outfile, binary, f2);
|
||||
double d1 = RandUniform();
|
||||
WriteBasicType(outfile, binary, d1);
|
||||
if (!binary && Rand()%2 == 0) outfile << "\t";
|
||||
double d2 = RandUniform();
|
||||
WriteBasicType(outfile, binary, d2);
|
||||
if (!binary && Rand()%2 == 0) outfile << "\t";
|
||||
bool ans = ko.Close();
|
||||
KALDI_ASSERT(ans);
|
||||
#ifndef _MSC_VER
|
||||
Sleep(1); // This test does not work without this sleep:
|
||||
// seems to be some kind of file-system latency.
|
||||
#endif
|
||||
{
|
||||
bool binary_in;
|
||||
Input ki(filename_in, &binary_in);
|
||||
std::istream &infile = ki.Stream();
|
||||
int64 i1_in;
|
||||
ReadBasicType(infile, binary_in, &i1_in);
|
||||
KALDI_ASSERT(i1_in == i1);
|
||||
uint16 i2_in;
|
||||
ReadBasicType(infile, binary_in, &i2_in);
|
||||
KALDI_ASSERT(i2_in == i2);
|
||||
char c_in;
|
||||
ReadBasicType(infile, binary_in, &c_in);
|
||||
KALDI_ASSERT(c_in == c);
|
||||
std::vector<int32> vec1_in;
|
||||
ReadIntegerVector(infile, binary_in, &vec1_in);
|
||||
KALDI_ASSERT(vec1_in == vec1);
|
||||
std::vector<uint16> vec2_in;
|
||||
ReadIntegerVector(infile, binary_in, &vec2_in);
|
||||
KALDI_ASSERT(vec2_in == vec2);
|
||||
std::vector<char> vec3_in;
|
||||
KALDI_ASSERT(PeekToken(infile, binary_in) == static_cast<int>('f'));
|
||||
ExpectToken(infile, binary_in, "<foo>");
|
||||
ReadIntegerVector(infile, binary_in, &vec3_in);
|
||||
KALDI_ASSERT(vec3_in == vec3);
|
||||
std::string token1_in, token2_in;
|
||||
KALDI_ASSERT(Peek(infile, binary_in) == static_cast<int>(*token1));
|
||||
ReadToken(infile, binary_in, &token1_in);
|
||||
KALDI_ASSERT(token1_in == (std::string)token1);
|
||||
ReadToken(infile, binary_in, &token2_in);
|
||||
KALDI_ASSERT(token2_in == token2);
|
||||
if (Rand() % 2 == 0)
|
||||
ExpectToken(infile, binary_in, token3.c_str());
|
||||
else
|
||||
ExpectToken(infile, binary_in, token3);
|
||||
float f1_in; // same type.
|
||||
ReadBasicType(infile, binary_in, &f1_in);
|
||||
AssertEqual(f1_in, f1);
|
||||
double f2_in; // wrong type.
|
||||
ReadBasicType(infile, binary_in, &f2_in);
|
||||
AssertEqual(f2_in, f2);
|
||||
double d1_in; // same type.
|
||||
ReadBasicType(infile, binary_in, &d1_in);
|
||||
AssertEqual(d1_in, d1);
|
||||
float d2_in; // wrong type.
|
||||
ReadBasicType(infile, binary_in, &d2_in);
|
||||
AssertEqual(d2_in, d2);
|
||||
KALDI_ASSERT(Peek(infile, binary_in) == -1);
|
||||
}
|
||||
}
|
||||
unlink("tmpf.txt");
|
||||
unlink("tmpf.gz");
|
||||
}
|
||||
|
||||
void UnitTestIoStandard() {
|
||||
/*
|
||||
Don't do the the following part because it requires
|
||||
to pipe from an empty file, for it to not hang.
|
||||
{
|
||||
Input inp("", NULL); // standard input.
|
||||
KALDI_ASSERT(inp.Stream().get() == -1);
|
||||
}
|
||||
{
|
||||
Input inp("-", NULL); // standard input.
|
||||
KALDI_ASSERT(inp.Stream().get() == -1);
|
||||
}*/
|
||||
|
||||
{
|
||||
std::cout << "Should see: foo\n";
|
||||
Output out("", false);
|
||||
out.Stream() << "foo\n";
|
||||
}
|
||||
{
|
||||
std::cout << "Should see: bar\n";
|
||||
Output out("-", false);
|
||||
out.Stream() << "bar\n";
|
||||
}
|
||||
}
|
||||
|
||||
// This is Windows-specific.
|
||||
void UnitTestNativeFilename() {
|
||||
#ifdef KALDI_CYGWIN_COMPAT
|
||||
extern std::string MapCygwinPath(const std::string &filename);
|
||||
|
||||
KALDI_ASSERT(MapCygwinPath("") == "");
|
||||
KALDI_ASSERT(MapCygwinPath(".") == ".");
|
||||
KALDI_ASSERT(MapCygwinPath("..") == "..");
|
||||
KALDI_ASSERT(MapCygwinPath("/dev/null")[0] != '/');
|
||||
KALDI_ASSERT(MapCygwinPath("/tmp")[1] == ':');
|
||||
KALDI_ASSERT(MapCygwinPath("/tmp/")[1] == ':');
|
||||
KALDI_ASSERT(MapCygwinPath("/tmp/foo")[1] == ':');
|
||||
KALDI_ASSERT(MapCygwinPath("/cygdrive/c") == "c:/");
|
||||
KALDI_ASSERT(MapCygwinPath("/cygdrive/c/") == "c:/");
|
||||
KALDI_ASSERT(MapCygwinPath("/cygdrive/c/foo") == "c:/foo");
|
||||
#endif
|
||||
}
|
||||
|
||||
} // end namespace kaldi.
|
||||
|
||||
#if defined(_MSC_VER) && !defined(KALDI_CYGWIN_COMPAT)
|
||||
// Windows has no cat! There is probably no suitable tool to test popen I/O on
|
||||
// Windows, so we emulate a lame version of cat(1).
|
||||
static int TinyCat(int argc, const char** argv) {
|
||||
const char* name_in = argc > 0 && strcmp(argv[0], "-") ? argv[0] : NULL;
|
||||
int fd_in = name_in ? _open(name_in, _O_RDONLY) : _fileno(stdin);
|
||||
if (fd_in < 0)
|
||||
return 1;
|
||||
|
||||
int fd_out = _fileno(stdout);
|
||||
_setmode(fd_in, _O_BINARY);
|
||||
_setmode(fd_out, _O_BINARY);
|
||||
|
||||
char buffer[100];
|
||||
int last_read;
|
||||
while ((last_read = _read(fd_in, buffer, sizeof(buffer))) > 0)
|
||||
_write(fd_out, buffer, last_read);
|
||||
|
||||
if (name_in) _close(fd_in);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
int main(int argc, const char** argv) {
|
||||
using namespace kaldi;
|
||||
#if defined(_MSC_VER) && !defined(KALDI_CYGWIN_COMPAT)
|
||||
if (argc > 1 && strcmp(argv[1], "cat") == 0)
|
||||
return TinyCat(argc - 2, argv + 2);
|
||||
#endif
|
||||
|
||||
UnitTestNativeFilename();
|
||||
UnitTestIoNew(false);
|
||||
UnitTestIoNew(true);
|
||||
UnitTestIoPipe(true);
|
||||
UnitTestIoPipe(false);
|
||||
UnitTestIoStandard();
|
||||
UnitTestClassifyRxfilename();
|
||||
UnitTestClassifyWxfilename();
|
||||
|
||||
KALDI_ASSERT(1); // just wanted to check that KALDI_ASSERT does not fail
|
||||
// for 1.
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,880 @@
|
||||
// util/kaldi-io.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation; Jan Silovsky
|
||||
// 2016 Xiaohui Zhang
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "util/kaldi-io.h"
|
||||
#include <errno.h>
|
||||
#include <cstdlib>
|
||||
#include "base/kaldi-math.h"
|
||||
#include "util/text-utils.h"
|
||||
#include "util/parse-options.h"
|
||||
#include "util/kaldi-pipebuf.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#ifdef KALDI_CYGWIN_COMPAT
|
||||
#include "util/kaldi-cygwin-io-inl.h"
|
||||
#define MapOsPath(x) MapCygwinPath(x)
|
||||
#else // KALDI_CYGWIN_COMPAT
|
||||
#define MapOsPath(x) x
|
||||
#endif // KALDI_CYGWIN_COMPAT
|
||||
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
static FILE *popen(const char* command, const char* mode) {
|
||||
#ifdef KALDI_CYGWIN_COMPAT
|
||||
return kaldi::CygwinCompatPopen(command, mode);
|
||||
#else // KALDI_CYGWIN_COMPAT
|
||||
return _popen(command, mode);
|
||||
#endif // KALDI_CYGWIN_COMPAT
|
||||
}
|
||||
#endif // _MSC_VER
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
#ifndef _MSC_VER // on VS, we don't need this type.
|
||||
// could replace basic_pipebuf<char> with stdio_filebuf<char> on some platforms.
|
||||
// Would mean we could use less of our own code.
|
||||
typedef basic_pipebuf<char> PipebufType;
|
||||
#endif
|
||||
}
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
std::string PrintableRxfilename(const std::string &rxfilename) {
|
||||
if (rxfilename == "" || rxfilename == "-") {
|
||||
return "standard input";
|
||||
} else {
|
||||
// If this call to Escape later causes compilation issues,
|
||||
// just replace it with "return rxfilename"; it's only a
|
||||
// pretty-printing issue.
|
||||
return ParseOptions::Escape(rxfilename);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
std::string PrintableWxfilename(const std::string &wxfilename) {
|
||||
if (wxfilename == "" || wxfilename == "-") {
|
||||
return "standard output";
|
||||
} else {
|
||||
// If this call to Escape later causes compilation issues,
|
||||
// just replace it with "return wxfilename"; it's only a
|
||||
// pretty-printing issue.
|
||||
return ParseOptions::Escape(wxfilename);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
OutputType ClassifyWxfilename(const std::string &filename) {
|
||||
const char *c = filename.c_str();
|
||||
size_t length = filename.length();
|
||||
char first_char = c[0],
|
||||
last_char = (length == 0 ? '\0' : c[filename.length()-1]);
|
||||
|
||||
// if 'filename' is "" or "-", return kStandardOutput.
|
||||
if (length == 0 || (length == 1 && first_char == '-'))
|
||||
return kStandardOutput;
|
||||
else if (first_char == '|') return kPipeOutput; // An output pipe like "|blah".
|
||||
else if (isspace(first_char) || isspace(last_char) || last_char == '|') {
|
||||
return kNoOutput; // Leading or trailing space: can't interpret this.
|
||||
// Final '|' would represent an input pipe, not an
|
||||
// output pipe.
|
||||
//} else if ((first_char == 'a' || first_char == 's') &&
|
||||
// strchr(c, ':') != NULL &&
|
||||
// (ClassifyWspecifier(filename, NULL, NULL, NULL) != kNoWspecifier ||
|
||||
// ClassifyRspecifier(filename, NULL, NULL) != kNoRspecifier)) {
|
||||
// // e.g. ark:something or scp:something... this is almost certainly a
|
||||
// // scripting error, so call it an error rather than treating it as a file.
|
||||
// // In practice in modern kaldi scripts all (r,w)filenames begin with "ark"
|
||||
// // or "scp", even though technically speaking options like "b", "t", "s" or
|
||||
// // "cs" can appear before the ark or scp, like "b,ark". For efficiency,
|
||||
// // and because this code is really just a nicety to catch errors earlier
|
||||
// // than they would otherwise be caught, we only call those extra functions
|
||||
// // for filenames beginning with 'a' or 's'.
|
||||
// return kNoOutput;
|
||||
} else if (isdigit(last_char)) {
|
||||
// This could be a file, but we have to see if it's an offset into a file
|
||||
// (like foo.ark:4314328), which is not allowed for writing (but is
|
||||
// allowed for reaching). This eliminates some things which would be
|
||||
// valid UNIX filenames but are not allowed by Kaldi. (Even if we allowed
|
||||
// such filenames for writing, we woudln't be able to correctly read them).
|
||||
const char *d = c + length - 1;
|
||||
while (isdigit(*d) && d > c) d--;
|
||||
if (*d == ':') return kNoOutput;
|
||||
// else it could still be a filename; continue to the next check.
|
||||
}
|
||||
|
||||
// At this point it matched no other pattern so we assume a filename, but we
|
||||
// check for internal '|' as it's a common source of errors to have pipe
|
||||
// commands without the pipe in the right place. Say that it can't be
|
||||
// classified.
|
||||
if (strchr(c, '|') != NULL) {
|
||||
KALDI_WARN << "Trying to classify wxfilename with pipe symbol in the"
|
||||
" wrong place (pipe without | at the beginning?): " <<
|
||||
filename;
|
||||
return kNoOutput;
|
||||
}
|
||||
return kFileOutput; // It matched no other pattern: assume it's a filename.
|
||||
}
|
||||
|
||||
|
||||
InputType ClassifyRxfilename(const std::string &filename) {
|
||||
const char *c = filename.c_str();
|
||||
size_t length = filename.length();
|
||||
char first_char = c[0],
|
||||
last_char = (length == 0 ? '\0' : c[filename.length()-1]);
|
||||
|
||||
// if 'filename' is "" or "-", return kStandardInput.
|
||||
if (length == 0 || (length == 1 && first_char == '-')) {
|
||||
return kStandardInput;
|
||||
} else if (first_char == '|') {
|
||||
return kNoInput; // An output pipe like "|blah": not
|
||||
// valid for input.
|
||||
} else if (last_char == '|') {
|
||||
return kPipeInput;
|
||||
} else if (isspace(first_char) || isspace(last_char)) {
|
||||
return kNoInput; // We don't allow leading or trailing space in a filename.
|
||||
//} else if ((first_char == 'a' || first_char == 's') &&
|
||||
// strchr(c, ':') != NULL &&
|
||||
// (ClassifyWspecifier(filename, NULL, NULL, NULL) != kNoWspecifier ||
|
||||
// ClassifyRspecifier(filename, NULL, NULL) != kNoRspecifier)) {
|
||||
// // e.g. ark:something or scp:something... this is almost certainly a
|
||||
// // scripting error, so call it an error rather than treating it as a file.
|
||||
// // In practice in modern kaldi scripts all (r,w)filenames begin with "ark"
|
||||
// // or "scp", even though technically speaking options like "b", "t", "s" or
|
||||
// // "cs" can appear before the ark or scp, like "b,ark". For efficiency,
|
||||
// // and because this code is really just a nicety to catch errors earlier
|
||||
// // than they would otherwise be caught, we only call those extra functions
|
||||
// // for filenames beginning with 'a' or 's'.
|
||||
// return kNoInput;
|
||||
} else if (isdigit(last_char)) {
|
||||
const char *d = c + length - 1;
|
||||
while (isdigit(*d) && d > c) d--;
|
||||
if (*d == ':') return kOffsetFileInput; // Filename is like
|
||||
// some_file:12345
|
||||
// otherwise it could still be a filename; continue to the next check.
|
||||
}
|
||||
|
||||
|
||||
// At this point it matched no other pattern so we assume a filename, but
|
||||
// we check for '|' as it's a common source of errors to have pipe
|
||||
// commands without the pipe in the right place. Say that it can't be
|
||||
// classified in this case.
|
||||
if (strchr(c, '|') != NULL) {
|
||||
KALDI_WARN << "Trying to classify rxfilename with pipe symbol in the"
|
||||
" wrong place (pipe without | at the end?): " << filename;
|
||||
return kNoInput;
|
||||
}
|
||||
return kFileInput; // It matched no other pattern: assume it's a filename.
|
||||
}
|
||||
|
||||
class OutputImplBase {
|
||||
public:
|
||||
// Open will open it as a file (no header), and return true
|
||||
// on success. It cannot be called on an already open stream.
|
||||
virtual bool Open(const std::string &filename, bool binary) = 0;
|
||||
virtual std::ostream &Stream() = 0;
|
||||
virtual bool Close() = 0;
|
||||
virtual ~OutputImplBase() { }
|
||||
};
|
||||
|
||||
|
||||
class FileOutputImpl: public OutputImplBase {
|
||||
public:
|
||||
virtual bool Open(const std::string &filename, bool binary) {
|
||||
if (os_.is_open()) KALDI_ERR << "FileOutputImpl::Open(), "
|
||||
<< "open called on already open file.";
|
||||
filename_ = filename;
|
||||
os_.open(MapOsPath(filename_).c_str(),
|
||||
binary ? std::ios_base::out | std::ios_base::binary
|
||||
: std::ios_base::out);
|
||||
return os_.is_open();
|
||||
}
|
||||
|
||||
virtual std::ostream &Stream() {
|
||||
if (!os_.is_open())
|
||||
KALDI_ERR << "FileOutputImpl::Stream(), file is not open.";
|
||||
// I believe this error can only arise from coding error.
|
||||
return os_;
|
||||
}
|
||||
|
||||
virtual bool Close() {
|
||||
if (!os_.is_open())
|
||||
KALDI_ERR << "FileOutputImpl::Close(), file is not open.";
|
||||
// I believe this error can only arise from coding error.
|
||||
os_.close();
|
||||
return !(os_.fail());
|
||||
}
|
||||
virtual ~FileOutputImpl() {
|
||||
if (os_.is_open()) {
|
||||
os_.close();
|
||||
if (os_.fail())
|
||||
KALDI_ERR << "Error closing output file " << filename_;
|
||||
}
|
||||
}
|
||||
private:
|
||||
std::string filename_;
|
||||
std::ofstream os_;
|
||||
};
|
||||
|
||||
class StandardOutputImpl: public OutputImplBase {
|
||||
public:
|
||||
StandardOutputImpl(): is_open_(false) { }
|
||||
|
||||
virtual bool Open(const std::string &filename, bool binary) {
|
||||
if (is_open_) KALDI_ERR << "StandardOutputImpl::Open(), "
|
||||
"open called on already open file.";
|
||||
#ifdef _MSC_VER
|
||||
_setmode(_fileno(stdout), binary ? _O_BINARY : _O_TEXT);
|
||||
#endif
|
||||
is_open_ = std::cout.good();
|
||||
return is_open_;
|
||||
}
|
||||
|
||||
virtual std::ostream &Stream() {
|
||||
if (!is_open_)
|
||||
KALDI_ERR << "StandardOutputImpl::Stream(), object not initialized.";
|
||||
// I believe this error can only arise from coding error.
|
||||
return std::cout;
|
||||
}
|
||||
|
||||
virtual bool Close() {
|
||||
if (!is_open_)
|
||||
KALDI_ERR << "StandardOutputImpl::Close(), file is not open.";
|
||||
is_open_ = false;
|
||||
std::cout << std::flush;
|
||||
return !(std::cout.fail());
|
||||
}
|
||||
virtual ~StandardOutputImpl() {
|
||||
if (is_open_) {
|
||||
std::cout << std::flush;
|
||||
if (std::cout.fail())
|
||||
KALDI_ERR << "Error writing to standard output";
|
||||
}
|
||||
}
|
||||
private:
|
||||
bool is_open_;
|
||||
};
|
||||
|
||||
class PipeOutputImpl: public OutputImplBase {
|
||||
public:
|
||||
PipeOutputImpl(): f_(NULL), os_(NULL) { }
|
||||
|
||||
virtual bool Open(const std::string &wxfilename, bool binary) {
|
||||
filename_ = wxfilename;
|
||||
KALDI_ASSERT(f_ == NULL); // Make sure closed.
|
||||
KALDI_ASSERT(wxfilename.length() != 0 && wxfilename[0] == '|'); // should
|
||||
// start with '|'
|
||||
std::string cmd_name(wxfilename, 1);
|
||||
#if defined(_MSC_VER) || defined(__CYGWIN__)
|
||||
f_ = popen(cmd_name.c_str(), (binary ? "wb" : "w"));
|
||||
#else
|
||||
f_ = popen(cmd_name.c_str(), "w");
|
||||
#endif
|
||||
if (!f_) { // Failure.
|
||||
KALDI_WARN << "Failed opening pipe for writing, command is: "
|
||||
<< cmd_name << ", errno is " << strerror(errno);
|
||||
return false;
|
||||
} else {
|
||||
#ifndef _MSC_VER
|
||||
fb_ = new PipebufType(f_, // Using this constructor won't make the
|
||||
// destructor try to close the stream when
|
||||
// we're done.
|
||||
(binary ? std::ios_base::out|
|
||||
std::ios_base::binary
|
||||
:std::ios_base::out));
|
||||
KALDI_ASSERT(fb_ != NULL); // or would be alloc error.
|
||||
os_ = new std::ostream(fb_);
|
||||
#else
|
||||
os_ = new std::ofstream(f_);
|
||||
#endif
|
||||
return os_->good();
|
||||
}
|
||||
}
|
||||
|
||||
virtual std::ostream &Stream() {
|
||||
if (os_ == NULL) KALDI_ERR << "PipeOutputImpl::Stream(),"
|
||||
" object not initialized.";
|
||||
// I believe this error can only arise from coding error.
|
||||
return *os_;
|
||||
}
|
||||
|
||||
virtual bool Close() {
|
||||
if (os_ == NULL) KALDI_ERR << "PipeOutputImpl::Close(), file is not open.";
|
||||
bool ok = true;
|
||||
os_->flush();
|
||||
if (os_->fail()) ok = false;
|
||||
delete os_;
|
||||
os_ = NULL;
|
||||
int status;
|
||||
#ifdef _MSC_VER
|
||||
status = _pclose(f_);
|
||||
#else
|
||||
status = pclose(f_);
|
||||
#endif
|
||||
if (status)
|
||||
KALDI_WARN << "Pipe " << filename_ << " had nonzero return status "
|
||||
<< status;
|
||||
f_ = NULL;
|
||||
#ifndef _MSC_VER
|
||||
delete fb_;
|
||||
fb_ = NULL;
|
||||
#endif
|
||||
return ok;
|
||||
}
|
||||
virtual ~PipeOutputImpl() {
|
||||
if (os_) {
|
||||
if (!Close())
|
||||
KALDI_ERR << "Error writing to pipe " << PrintableWxfilename(filename_);
|
||||
}
|
||||
}
|
||||
private:
|
||||
std::string filename_;
|
||||
FILE *f_;
|
||||
#ifndef _MSC_VER
|
||||
PipebufType *fb_;
|
||||
#endif
|
||||
std::ostream *os_;
|
||||
};
|
||||
|
||||
|
||||
|
||||
class InputImplBase {
|
||||
public:
|
||||
// Open will open it as a file, and return true on success.
|
||||
// May be called twice only for kOffsetFileInput (otherwise,
|
||||
// if called twice, we just create a new Input object, to avoid
|
||||
// having to deal with the extra hassle of reopening with the
|
||||
// same object.
|
||||
// Note that we will to call Open with true (binary) for
|
||||
// for text-mode Kaldi files; the only actual text-mode input
|
||||
// is for non-Kaldi files.
|
||||
virtual bool Open(const std::string &filename, bool binary) = 0;
|
||||
virtual std::istream &Stream() = 0;
|
||||
virtual int32 Close() = 0; // We only need to check failure in the case of
|
||||
// kPipeInput.
|
||||
// on close for input streams.
|
||||
virtual InputType MyType() = 0; // Because if it's kOffsetFileInput, we may
|
||||
// call Open twice
|
||||
// (has efficiency benefits).
|
||||
|
||||
virtual ~InputImplBase() { }
|
||||
};
|
||||
|
||||
class FileInputImpl: public InputImplBase {
|
||||
public:
|
||||
virtual bool Open(const std::string &filename, bool binary) {
|
||||
if (is_.is_open()) KALDI_ERR << "FileInputImpl::Open(), "
|
||||
<< "open called on already open file.";
|
||||
is_.open(MapOsPath(filename).c_str(),
|
||||
binary ? std::ios_base::in | std::ios_base::binary
|
||||
: std::ios_base::in);
|
||||
return is_.is_open();
|
||||
}
|
||||
|
||||
virtual std::istream &Stream() {
|
||||
if (!is_.is_open())
|
||||
KALDI_ERR << "FileInputImpl::Stream(), file is not open.";
|
||||
// I believe this error can only arise from coding error.
|
||||
return is_;
|
||||
}
|
||||
|
||||
virtual int32 Close() {
|
||||
if (!is_.is_open())
|
||||
KALDI_ERR << "FileInputImpl::Close(), file is not open.";
|
||||
// I believe this error can only arise from coding error.
|
||||
is_.close();
|
||||
// Don't check status.
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual InputType MyType() { return kFileInput; }
|
||||
|
||||
virtual ~FileInputImpl() {
|
||||
// Stream will automatically be closed, and we don't care about
|
||||
// whether it fails.
|
||||
}
|
||||
private:
|
||||
std::ifstream is_;
|
||||
};
|
||||
|
||||
|
||||
class StandardInputImpl: public InputImplBase {
|
||||
public:
|
||||
StandardInputImpl(): is_open_(false) { }
|
||||
|
||||
virtual bool Open(const std::string &filename, bool binary) {
|
||||
if (is_open_) KALDI_ERR << "StandardInputImpl::Open(), "
|
||||
"open called on already open file.";
|
||||
is_open_ = true;
|
||||
#ifdef _MSC_VER
|
||||
_setmode(_fileno(stdin), binary ? _O_BINARY : _O_TEXT);
|
||||
#endif
|
||||
return true; // Don't check good() because would be false if
|
||||
// eof, which may be valid input.
|
||||
}
|
||||
|
||||
virtual std::istream &Stream() {
|
||||
if (!is_open_)
|
||||
KALDI_ERR << "StandardInputImpl::Stream(), object not initialized.";
|
||||
// I believe this error can only arise from coding error.
|
||||
return std::cin;
|
||||
}
|
||||
|
||||
virtual InputType MyType() { return kStandardInput; }
|
||||
|
||||
virtual int32 Close() {
|
||||
if (!is_open_) KALDI_ERR << "StandardInputImpl::Close(), file is not open.";
|
||||
is_open_ = false;
|
||||
return 0;
|
||||
}
|
||||
virtual ~StandardInputImpl() { }
|
||||
private:
|
||||
bool is_open_;
|
||||
};
|
||||
|
||||
class PipeInputImpl: public InputImplBase {
|
||||
public:
|
||||
PipeInputImpl(): f_(NULL), is_(NULL) { }
|
||||
|
||||
virtual bool Open(const std::string &rxfilename, bool binary) {
|
||||
filename_ = rxfilename;
|
||||
KALDI_ASSERT(f_ == NULL); // Make sure closed.
|
||||
KALDI_ASSERT(rxfilename.length() != 0 &&
|
||||
rxfilename[rxfilename.length()-1] == '|'); // should end with '|'
|
||||
std::string cmd_name(rxfilename, 0, rxfilename.length()-1);
|
||||
#if defined(_MSC_VER) || defined(__CYGWIN__)
|
||||
f_ = popen(cmd_name.c_str(), (binary ? "rb" : "r"));
|
||||
#else
|
||||
f_ = popen(cmd_name.c_str(), "r");
|
||||
#endif
|
||||
|
||||
if (!f_) { // Failure.
|
||||
KALDI_WARN << "Failed opening pipe for reading, command is: "
|
||||
<< cmd_name << ", errno is " << strerror(errno);
|
||||
return false;
|
||||
} else {
|
||||
#ifndef _MSC_VER
|
||||
fb_ = new PipebufType(f_, // Using this constructor won't lead the
|
||||
// destructor to close the stream.
|
||||
(binary ? std::ios_base::in|
|
||||
std::ios_base::binary
|
||||
:std::ios_base::in));
|
||||
KALDI_ASSERT(fb_ != NULL); // or would be alloc error.
|
||||
is_ = new std::istream(fb_);
|
||||
#else
|
||||
is_ = new std::ifstream(f_);
|
||||
#endif
|
||||
if (is_->fail() || is_->bad()) return false;
|
||||
if (is_->eof()) {
|
||||
KALDI_WARN << "Pipe opened with command "
|
||||
<< PrintableRxfilename(rxfilename)
|
||||
<< " is empty.";
|
||||
// don't return false: empty may be valid.
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
virtual std::istream &Stream() {
|
||||
if (is_ == NULL)
|
||||
KALDI_ERR << "PipeInputImpl::Stream(), object not initialized.";
|
||||
// I believe this error can only arise from coding error.
|
||||
return *is_;
|
||||
}
|
||||
|
||||
virtual int32 Close() {
|
||||
if (is_ == NULL)
|
||||
KALDI_ERR << "PipeInputImpl::Close(), file is not open.";
|
||||
delete is_;
|
||||
is_ = NULL;
|
||||
int32 status;
|
||||
#ifdef _MSC_VER
|
||||
status = _pclose(f_);
|
||||
#else
|
||||
status = pclose(f_);
|
||||
#endif
|
||||
if (status)
|
||||
KALDI_WARN << "Pipe " << filename_ << " had nonzero return status "
|
||||
<< status;
|
||||
f_ = NULL;
|
||||
#ifndef _MSC_VER
|
||||
delete fb_;
|
||||
fb_ = NULL;
|
||||
#endif
|
||||
return status;
|
||||
}
|
||||
virtual ~PipeInputImpl() {
|
||||
if (is_)
|
||||
Close();
|
||||
}
|
||||
virtual InputType MyType() { return kPipeInput; }
|
||||
private:
|
||||
std::string filename_;
|
||||
FILE *f_;
|
||||
#ifndef _MSC_VER
|
||||
PipebufType *fb_;
|
||||
#endif
|
||||
std::istream *is_;
|
||||
};
|
||||
|
||||
/*
|
||||
#else
|
||||
|
||||
// Just have an empty implementation of the pipe input that crashes if
|
||||
// called.
|
||||
class PipeInputImpl: public InputImplBase {
|
||||
public:
|
||||
PipeInputImpl() { KALDI_ASSERT(0 && "Pipe input not yet supported on this
|
||||
platform."); }
|
||||
virtual bool Open(const std::string, bool) { return 0; }
|
||||
virtual std::istream &Stream() const { return NULL; }
|
||||
virtual void Close() {}
|
||||
virtual InputType MyType() { return kPipeInput; }
|
||||
};
|
||||
|
||||
#endif
|
||||
*/
|
||||
|
||||
class OffsetFileInputImpl: public InputImplBase {
|
||||
// This class is a bit more complicated than the
|
||||
|
||||
public:
|
||||
// splits a filename like /my/file:123 into /my/file and the
|
||||
// number 123. Crashes if not this format.
|
||||
static void SplitFilename(const std::string &rxfilename,
|
||||
std::string *filename,
|
||||
size_t *offset) {
|
||||
size_t pos = rxfilename.find_last_of(':');
|
||||
KALDI_ASSERT(pos != std::string::npos); // would indicate error in calling
|
||||
// code, as the filename is supposed to be of the correct form at this
|
||||
// point.
|
||||
*filename = std::string(rxfilename, 0, pos);
|
||||
std::string number(rxfilename, pos+1);
|
||||
bool ans = ConvertStringToInteger(number, offset);
|
||||
if (!ans)
|
||||
KALDI_ERR << "Cannot get offset from filename " << rxfilename
|
||||
<< " (possibly you compiled in 32-bit and have a >32-bit"
|
||||
<< " byte offset into a file; you'll have to compile 64-bit.";
|
||||
}
|
||||
|
||||
bool Seek(size_t offset) {
|
||||
size_t cur_pos = is_.tellg();
|
||||
if (cur_pos == offset) return true;
|
||||
else if (cur_pos<offset && cur_pos+100 > offset) {
|
||||
// We're close enough that it may be faster to just
|
||||
// read that data, rather than seek.
|
||||
for (size_t i = cur_pos; i < offset; i++)
|
||||
is_.get();
|
||||
return (is_.tellg() == std::streampos(offset));
|
||||
}
|
||||
// Try to actually seek.
|
||||
is_.seekg(offset, std::ios_base::beg);
|
||||
if (is_.fail()) { // failbit or badbit is set [error happened]
|
||||
is_.close();
|
||||
return false; // failure.
|
||||
} else {
|
||||
is_.clear(); // Clear any failure bits (e.g. eof).
|
||||
return true; // success.
|
||||
}
|
||||
}
|
||||
|
||||
// This Open routine is unusual in that it is designed to work even
|
||||
// if it was already open. This for efficiency when seeking multiple
|
||||
// times.
|
||||
virtual bool Open(const std::string &rxfilename, bool binary) {
|
||||
if (is_.is_open()) {
|
||||
// We are opening when we have an already-open file.
|
||||
// We may have to seek within this file, or else close it and
|
||||
// open a different one.
|
||||
std::string tmp_filename;
|
||||
size_t offset;
|
||||
SplitFilename(rxfilename, &tmp_filename, &offset);
|
||||
if (tmp_filename == filename_ && binary == binary_) { // Just seek
|
||||
is_.clear(); // clear fail bit, etc.
|
||||
return Seek(offset);
|
||||
} else {
|
||||
is_.close(); // don't bother checking error status of is_.
|
||||
filename_ = tmp_filename;
|
||||
is_.open(MapOsPath(filename_).c_str(),
|
||||
binary ? std::ios_base::in | std::ios_base::binary
|
||||
: std::ios_base::in);
|
||||
if (!is_.is_open()) return false;
|
||||
else
|
||||
return Seek(offset);
|
||||
}
|
||||
} else {
|
||||
size_t offset;
|
||||
SplitFilename(rxfilename, &filename_, &offset);
|
||||
binary_ = binary;
|
||||
is_.open(MapOsPath(filename_).c_str(),
|
||||
binary ? std::ios_base::in | std::ios_base::binary
|
||||
: std::ios_base::in);
|
||||
if (!is_.is_open()) return false;
|
||||
else
|
||||
return Seek(offset);
|
||||
}
|
||||
}
|
||||
|
||||
virtual std::istream &Stream() {
|
||||
if (!is_.is_open())
|
||||
KALDI_ERR << "FileInputImpl::Stream(), file is not open.";
|
||||
// I believe this error can only arise from coding error.
|
||||
return is_;
|
||||
}
|
||||
|
||||
virtual int32 Close() {
|
||||
if (!is_.is_open())
|
||||
KALDI_ERR << "FileInputImpl::Close(), file is not open.";
|
||||
// I believe this error can only arise from coding error.
|
||||
is_.close();
|
||||
// Don't check status.
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual InputType MyType() { return kOffsetFileInput; }
|
||||
|
||||
virtual ~OffsetFileInputImpl() {
|
||||
// Stream will automatically be closed, and we don't care about
|
||||
// whether it fails.
|
||||
}
|
||||
private:
|
||||
std::string filename_; // the actual filename
|
||||
bool binary_; // true if was opened in binary mode.
|
||||
std::ifstream is_;
|
||||
};
|
||||
|
||||
|
||||
Output::Output(const std::string &wxfilename, bool binary,
|
||||
bool write_header):impl_(NULL) {
|
||||
if (!Open(wxfilename, binary, write_header)) {
|
||||
if (impl_) {
|
||||
delete impl_;
|
||||
impl_ = NULL;
|
||||
}
|
||||
KALDI_ERR << "Error opening output stream " <<
|
||||
PrintableWxfilename(wxfilename);
|
||||
}
|
||||
}
|
||||
|
||||
bool Output::Close() {
|
||||
if (!impl_) {
|
||||
return false; // error to call Close if not open.
|
||||
} else {
|
||||
bool ans = impl_->Close();
|
||||
delete impl_;
|
||||
impl_ = NULL;
|
||||
return ans;
|
||||
}
|
||||
}
|
||||
|
||||
Output::~Output() {
|
||||
if (impl_) {
|
||||
bool ok = impl_->Close();
|
||||
delete impl_;
|
||||
impl_ = NULL;
|
||||
if (!ok)
|
||||
KALDI_ERR << "Error closing output file "
|
||||
<< PrintableWxfilename(filename_)
|
||||
<< (ClassifyWxfilename(filename_) == kFileOutput ?
|
||||
" (disk full?)" : "");
|
||||
}
|
||||
}
|
||||
|
||||
std::ostream &Output::Stream() { // will throw if not open; else returns
|
||||
// stream.
|
||||
if (!impl_) KALDI_ERR << "Output::Stream() called but not open.";
|
||||
return impl_->Stream();
|
||||
}
|
||||
|
||||
bool Output::Open(const std::string &wxfn, bool binary, bool header) {
|
||||
if (IsOpen()) {
|
||||
if (!Close()) { // Throw here rather than return status, as it's an error
|
||||
// about something else: if the user wanted to avoid the exception he/she
|
||||
// could have called Close().
|
||||
KALDI_ERR << "Output::Open(), failed to close output stream: "
|
||||
<< PrintableWxfilename(filename_);
|
||||
}
|
||||
}
|
||||
|
||||
filename_ = wxfn;
|
||||
|
||||
OutputType type = ClassifyWxfilename(wxfn);
|
||||
KALDI_ASSERT(impl_ == NULL);
|
||||
|
||||
if (type == kFileOutput) {
|
||||
impl_ = new FileOutputImpl();
|
||||
} else if (type == kStandardOutput) {
|
||||
impl_ = new StandardOutputImpl();
|
||||
} else if (type == kPipeOutput) {
|
||||
impl_ = new PipeOutputImpl();
|
||||
} else { // type == kNoOutput
|
||||
KALDI_WARN << "Invalid output filename format "<<
|
||||
PrintableWxfilename(wxfn);
|
||||
return false;
|
||||
}
|
||||
if (!impl_->Open(wxfn, binary)) {
|
||||
delete impl_;
|
||||
impl_ = NULL;
|
||||
return false; // failed to open.
|
||||
} else { // successfully opened it.
|
||||
if (header) {
|
||||
InitKaldiOutputStream(impl_->Stream(), binary);
|
||||
bool ok = impl_->Stream().good(); // still OK?
|
||||
if (!ok) {
|
||||
delete impl_;
|
||||
impl_ = NULL;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Input::Input(const std::string &rxfilename, bool *binary): impl_(NULL) {
|
||||
if (!Open(rxfilename, binary)) {
|
||||
KALDI_ERR << "Error opening input stream "
|
||||
<< PrintableRxfilename(rxfilename);
|
||||
}
|
||||
}
|
||||
|
||||
int32 Input::Close() {
|
||||
if (impl_) {
|
||||
int32 ans = impl_->Close();
|
||||
delete impl_;
|
||||
impl_ = NULL;
|
||||
return ans;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
bool Input::OpenInternal(const std::string &rxfilename,
|
||||
bool file_binary,
|
||||
bool *contents_binary) {
|
||||
InputType type = ClassifyRxfilename(rxfilename);
|
||||
if (IsOpen()) {
|
||||
// May have to close the stream first.
|
||||
if (type == kOffsetFileInput && impl_->MyType() == kOffsetFileInput) {
|
||||
// We want to use the same object to Open... this is in case
|
||||
// the files are the same, so we can just seek.
|
||||
if (!impl_->Open(rxfilename, file_binary)) { // true is binary mode--
|
||||
// always open in binary.
|
||||
delete impl_;
|
||||
impl_ = NULL;
|
||||
return false;
|
||||
}
|
||||
// read the binary header, if requested.
|
||||
if (contents_binary != NULL)
|
||||
return InitKaldiInputStream(impl_->Stream(), contents_binary);
|
||||
else
|
||||
return true;
|
||||
} else {
|
||||
Close();
|
||||
// and fall through to code below which actually opens the file.
|
||||
}
|
||||
}
|
||||
if (type == kFileInput) {
|
||||
impl_ = new FileInputImpl();
|
||||
} else if (type == kStandardInput) {
|
||||
impl_ = new StandardInputImpl();
|
||||
} else if (type == kPipeInput) {
|
||||
impl_ = new PipeInputImpl();
|
||||
} else if (type == kOffsetFileInput) {
|
||||
impl_ = new OffsetFileInputImpl();
|
||||
} else { // type == kNoInput
|
||||
KALDI_WARN << "Invalid input filename format "<<
|
||||
PrintableRxfilename(rxfilename);
|
||||
return false;
|
||||
}
|
||||
if (!impl_->Open(rxfilename, file_binary)) { // true is binary mode--
|
||||
// always read in binary.
|
||||
delete impl_;
|
||||
impl_ = NULL;
|
||||
return false;
|
||||
}
|
||||
if (contents_binary != NULL)
|
||||
return InitKaldiInputStream(impl_->Stream(), contents_binary);
|
||||
else
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
Input::~Input() { if (impl_) Close(); }
|
||||
|
||||
|
||||
std::istream &Input::Stream() {
|
||||
if (!IsOpen()) KALDI_ERR << "Input::Stream(), not open.";
|
||||
return impl_->Stream();
|
||||
}
|
||||
|
||||
|
||||
//template <> void ReadKaldiObject(const std::string &filename,
|
||||
// Matrix<float> *m) {
|
||||
// if (!filename.empty() && filename[filename.size() - 1] == ']') {
|
||||
// // This filename seems to have a 'range'... like foo.ark:4312423[20:30].
|
||||
// // (the bit in square brackets is the range).
|
||||
// std::string rxfilename, range;
|
||||
// if (!ExtractRangeSpecifier(filename, &rxfilename, &range)) {
|
||||
// KALDI_ERR << "Could not make sense of possible range specifier in filename "
|
||||
// << "while reading matrix: " << filename;
|
||||
// }
|
||||
// Matrix<float> temp;
|
||||
// bool binary_in;
|
||||
// Input ki(rxfilename, &binary_in);
|
||||
// temp.Read(ki.Stream(), binary_in);
|
||||
// if (!ExtractObjectRange(temp, range, m)) {
|
||||
// KALDI_ERR << "Error extracting range of object: " << filename;
|
||||
// }
|
||||
// } else {
|
||||
// // The normal case, there is no range.
|
||||
// bool binary_in;
|
||||
// Input ki(filename, &binary_in);
|
||||
// m->Read(ki.Stream(), binary_in);
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//template <> void ReadKaldiObject(const std::string &filename,
|
||||
// Matrix<double> *m) {
|
||||
// if (!filename.empty() && filename[filename.size() - 1] == ']') {
|
||||
// // This filename seems to have a 'range'... like foo.ark:4312423[20:30].
|
||||
// // (the bit in square brackets is the range).
|
||||
// std::string rxfilename, range;
|
||||
// if (!ExtractRangeSpecifier(filename, &rxfilename, &range)) {
|
||||
// KALDI_ERR << "Could not make sense of possible range specifier in filename "
|
||||
// << "while reading matrix: " << filename;
|
||||
// }
|
||||
// Matrix<double> temp;
|
||||
// bool binary_in;
|
||||
// Input ki(rxfilename, &binary_in);
|
||||
// temp.Read(ki.Stream(), binary_in);
|
||||
// if (!ExtractObjectRange(temp, range, m)) {
|
||||
// KALDI_ERR << "Error extracting range of object: " << filename;
|
||||
// }
|
||||
// } else {
|
||||
// // The normal case, there is no range.
|
||||
// bool binary_in;
|
||||
// Input ki(filename, &binary_in);
|
||||
// m->Read(ki.Stream(), binary_in);
|
||||
// }
|
||||
//}
|
||||
|
||||
} // end namespace kaldi
|
||||
@@ -0,0 +1,280 @@
|
||||
// util/kaldi-io.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation; Jan Silovsky
|
||||
// 2016 Xiaohui Zhang
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#ifndef KALDI_UTIL_KALDI_IO_H_
|
||||
#define KALDI_UTIL_KALDI_IO_H_
|
||||
|
||||
#ifdef _MSC_VER
|
||||
# include <fcntl.h>
|
||||
# include <io.h>
|
||||
#endif
|
||||
#include <cctype> // For isspace.
|
||||
#include <limits>
|
||||
#include <string>
|
||||
#include "base/kaldi-common.h"
|
||||
//#include "matrix/kaldi-matrix.h"
|
||||
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
class OutputImplBase; // Forward decl; defined in a .cc file
|
||||
class InputImplBase; // Forward decl; defined in a .cc file
|
||||
|
||||
/// \addtogroup io_group
|
||||
/// @{
|
||||
|
||||
// The Output and Input classes handle stream-opening for "extended" filenames
|
||||
// that include actual files, standard-input/standard-output, pipes, and
|
||||
// offsets into actual files. They also handle reading and writing the
|
||||
// binary-mode headers for Kaldi files, where applicable. The classes have
|
||||
// versions of the Open routines that throw and do not throw, depending whether
|
||||
// the calling code wants to catch the errors or not; there are also versions
|
||||
// that write (or do not write) the Kaldi binary-mode header that says if it's
|
||||
// binary mode. Generally files that contain Kaldi objects will have the header
|
||||
// on, so we know upon reading them whether they have the header. So you would
|
||||
// use the OpenWithHeader routines for these (or the constructor); but other
|
||||
// types of objects (e.g. FSTs) would have files without a header so you would
|
||||
// use OpenNoHeader.
|
||||
|
||||
// We now document the types of extended filenames that we use.
|
||||
//
|
||||
// A "wxfilename" is an extended filename for writing. It can take three forms:
|
||||
// (1) Filename: e.g. "/some/filename", "./a/b/c", "c:\Users\dpovey\My
|
||||
// Documents\\boo"
|
||||
// (whatever the actual file-system interprets)
|
||||
// (2) Standard output: "" or "-"
|
||||
// (3) A pipe: e.g. "| gzip -c > /tmp/abc.gz"
|
||||
//
|
||||
//
|
||||
// A "rxfilename" is an extended filename for reading. It can take four forms:
|
||||
// (1) An actual filename, whatever the file-system can read, e.g. "/my/file".
|
||||
// (2) Standard input: "" or "-"
|
||||
// (3) A pipe: e.g. "gunzip -c /tmp/abc.gz |"
|
||||
// (4) An offset into a file, e.g.: "/mnt/blah/data/1.ark:24871"
|
||||
// [these are created by the Table and TableWriter classes; I may also write
|
||||
// a program that creates them for arbitrary files]
|
||||
//
|
||||
|
||||
|
||||
// Typical usage:
|
||||
// ...
|
||||
// bool binary;
|
||||
// MyObject.Write(Output(some_filename, binary).Stream(), binary);
|
||||
//
|
||||
// ... more extensive example:
|
||||
// {
|
||||
// Output ko(some_filename, binary);
|
||||
// MyObject1.Write(ko.Stream(), binary);
|
||||
// MyObject2.Write(ko.Stream(), binary);
|
||||
// }
|
||||
|
||||
|
||||
|
||||
enum OutputType {
|
||||
kNoOutput,
|
||||
kFileOutput,
|
||||
kStandardOutput,
|
||||
kPipeOutput
|
||||
};
|
||||
|
||||
/// ClassifyWxfilename interprets filenames as follows:
|
||||
/// - kNoOutput: invalid filenames (leading or trailing space, things that look
|
||||
/// like wspecifiers and rspecifiers or like pipes to read from with leading
|
||||
/// |.
|
||||
/// - kFileOutput: Normal filenames
|
||||
/// - kStandardOutput: The empty string or "-", interpreted as standard output
|
||||
/// - kPipeOutput: pipes, e.g. "| gzip -c > /tmp/abc.gz"
|
||||
OutputType ClassifyWxfilename(const std::string &wxfilename);
|
||||
|
||||
enum InputType {
|
||||
kNoInput,
|
||||
kFileInput,
|
||||
kStandardInput,
|
||||
kOffsetFileInput,
|
||||
kPipeInput
|
||||
};
|
||||
|
||||
/// ClassifyRxfilenames interprets filenames for reading as follows:
|
||||
/// - kNoInput: invalid filenames (leading or trailing space, things that
|
||||
/// look like wspecifiers and rspecifiers or pipes to write to
|
||||
/// with trailing |.
|
||||
/// - kFileInput: normal filenames
|
||||
/// - kStandardInput: the empty string or "-"
|
||||
/// - kPipeInput: e.g. "gunzip -c /tmp/abc.gz |"
|
||||
/// - kOffsetFileInput: offsets into files, e.g. /some/filename:12970
|
||||
InputType ClassifyRxfilename(const std::string &rxfilename);
|
||||
|
||||
|
||||
class Output {
|
||||
public:
|
||||
// The normal constructor, provided for convenience.
|
||||
// Equivalent to calling with default constructor then Open()
|
||||
// with these arguments.
|
||||
Output(const std::string &filename, bool binary, bool write_header = true);
|
||||
|
||||
Output(): impl_(NULL) {}
|
||||
|
||||
/// This opens the stream, with the given mode (binary or text). It returns
|
||||
/// true on success and false on failure. However, it will throw if something
|
||||
/// was already open and could not be closed (to avoid this, call Close()
|
||||
/// first. if write_header == true and binary == true, it writes the Kaldi
|
||||
/// binary-mode header ('\0' then 'B'). You may call Open even if it is
|
||||
/// already open; it will close the existing stream and reopen (however if
|
||||
/// closing the old stream failed it will throw).
|
||||
bool Open(const std::string &wxfilename, bool binary, bool write_header);
|
||||
|
||||
inline bool IsOpen(); // return true if we have an open stream. Does not
|
||||
// imply stream is good for writing.
|
||||
|
||||
std::ostream &Stream(); // will throw if not open; else returns stream.
|
||||
|
||||
// Close closes the stream. Calling Close is never necessary unless you
|
||||
// want to avoid exceptions being thrown. There are times when calling
|
||||
// Close will hurt efficiency (basically, when using offsets into files,
|
||||
// and using the same Input object),
|
||||
// but most of the time the user won't be doing this directly, it will
|
||||
// be done in kaldi-table.{h, cc}, so you don't have to worry about it.
|
||||
bool Close();
|
||||
|
||||
// This will throw if stream could not be closed (to check error status,
|
||||
// call Close()).
|
||||
~Output();
|
||||
|
||||
private:
|
||||
OutputImplBase *impl_; // non-NULL if open.
|
||||
std::string filename_;
|
||||
KALDI_DISALLOW_COPY_AND_ASSIGN(Output);
|
||||
};
|
||||
|
||||
|
||||
// bool binary_in;
|
||||
// Input ki(some_filename, &binary_in);
|
||||
// MyObject.Read(ki.Stream(), binary_in);
|
||||
//
|
||||
// ... more extensive example:
|
||||
//
|
||||
// {
|
||||
// bool binary_in;
|
||||
// Input ki(some_filename, &binary_in);
|
||||
// MyObject1.Read(ki.Stream(), &binary_in);
|
||||
// MyObject2.Write(ki.Stream(), &binary_in);
|
||||
// }
|
||||
// Note that to catch errors you need to use try.. catch.
|
||||
// Input communicates errors by throwing exceptions.
|
||||
|
||||
|
||||
// Input interprets four kinds of filenames:
|
||||
// (1) Normal filenames
|
||||
// (2) The empty string or "-", interpreted as standard output
|
||||
// (3) A pipe: e.g. "gunzip -c /tmp/abc.gz |"
|
||||
// (4) Offsets into [real] files, e.g. "/my/filename:12049"
|
||||
// The last one has no correspondence in Output.
|
||||
|
||||
|
||||
class Input {
|
||||
public:
|
||||
/// The normal constructor. Opens the stream in binary mode.
|
||||
/// Equivalent to calling the default constructor followed by Open(); then, if
|
||||
/// binary != NULL, it calls ReadHeader(), putting the output in "binary"; it
|
||||
/// throws on error.
|
||||
Input(const std::string &rxfilename, bool *contents_binary = NULL);
|
||||
|
||||
Input(): impl_(NULL) {}
|
||||
|
||||
// Open opens the stream for reading (the mode, where relevant, is binary; use
|
||||
// OpenTextMode for text-mode, we made this a separate function rather than a
|
||||
// boolean argument, to avoid confusion with Kaldi's text/binary distinction,
|
||||
// since reading in the file system's text mode is unusual.) If
|
||||
// contents_binary != NULL, it reads the binary-mode header and puts it in the
|
||||
// "binary" variable. Returns true on success. If it returns false it will
|
||||
// not be open. You may call Open even if it is already open; it will close
|
||||
// the existing stream and reopen (however if closing the old stream failed it
|
||||
// will throw).
|
||||
inline bool Open(const std::string &rxfilename, bool *contents_binary = NULL);
|
||||
|
||||
// As Open but (if the file system has text/binary modes) opens in text mode;
|
||||
// you shouldn't ever have to use this as in Kaldi we read even text files in
|
||||
// binary mode (and ignore the \r).
|
||||
inline bool OpenTextMode(const std::string &rxfilename);
|
||||
|
||||
// Return true if currently open for reading and Stream() will
|
||||
// succeed. Does not guarantee that the stream is good.
|
||||
inline bool IsOpen();
|
||||
|
||||
// It is never necessary or helpful to call Close, except if
|
||||
// you are concerned about to many filehandles being open.
|
||||
// Close does not throw. It returns the exit code as int32
|
||||
// in the case of a pipe [kPipeInput], and always zero otherwise.
|
||||
int32 Close();
|
||||
|
||||
// Returns the underlying stream. Throws if !IsOpen()
|
||||
std::istream &Stream();
|
||||
|
||||
// Destructor does not throw: input streams may legitimately fail so we
|
||||
// don't worry about the status when we close them.
|
||||
~Input();
|
||||
private:
|
||||
bool OpenInternal(const std::string &rxfilename, bool file_binary,
|
||||
bool *contents_binary);
|
||||
InputImplBase *impl_;
|
||||
KALDI_DISALLOW_COPY_AND_ASSIGN(Input);
|
||||
};
|
||||
|
||||
template <class C> void ReadKaldiObject(const std::string &filename,
|
||||
C *c) {
|
||||
bool binary_in;
|
||||
Input ki(filename, &binary_in);
|
||||
c->Read(ki.Stream(), binary_in);
|
||||
}
|
||||
|
||||
// Specialize the template for reading matrices, because we want to be able to
|
||||
// support reading 'ranges' (row and column ranges), like foo.mat[10:20].
|
||||
//template <> void ReadKaldiObject(const std::string &filename,
|
||||
// Matrix<float> *m);
|
||||
//
|
||||
//
|
||||
//template <> void ReadKaldiObject(const std::string &filename,
|
||||
// Matrix<double> *m);
|
||||
|
||||
|
||||
|
||||
template <class C> inline void WriteKaldiObject(const C &c,
|
||||
const std::string &filename,
|
||||
bool binary) {
|
||||
Output ko(filename, binary);
|
||||
c.Write(ko.Stream(), binary);
|
||||
}
|
||||
|
||||
/// PrintableRxfilename turns the rxfilename into a more human-readable
|
||||
/// form for error reporting, i.e. it does quoting and escaping and
|
||||
/// replaces "" or "-" with "standard input".
|
||||
std::string PrintableRxfilename(const std::string &rxfilename);
|
||||
|
||||
/// PrintableWxfilename turns the wxfilename into a more human-readable
|
||||
/// form for error reporting, i.e. it does quoting and escaping and
|
||||
/// replaces "" or "-" with "standard output".
|
||||
std::string PrintableWxfilename(const std::string &wxfilename);
|
||||
|
||||
/// @}
|
||||
|
||||
} // end namespace kaldi.
|
||||
|
||||
#include "util/kaldi-io-inl.h"
|
||||
|
||||
#endif // KALDI_UTIL_KALDI_IO_H_
|
||||
@@ -0,0 +1,87 @@
|
||||
// util/kaldi-pipebuf.h
|
||||
|
||||
// Copyright 2009-2011 Ondrej Glembek
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
/** @file kaldi-pipebuf.h
|
||||
* This is an Kaldi C++ Library header.
|
||||
*/
|
||||
|
||||
#ifndef KALDI_UTIL_KALDI_PIPEBUF_H_
|
||||
#define KALDI_UTIL_KALDI_PIPEBUF_H_
|
||||
|
||||
#include<string>
|
||||
#if !defined(_LIBCPP_VERSION) // libc++
|
||||
#include <fstream>
|
||||
#else
|
||||
#include "util/basic-filebuf.h"
|
||||
#endif
|
||||
|
||||
namespace kaldi {
|
||||
// This class provides a way to initialize a filebuf with a FILE* pointer
|
||||
// directly; it will not close the file pointer when it is deleted.
|
||||
// The C++ standard does not allow implementations of C++ to provide
|
||||
// this constructor within basic_filebuf, which makes it hard to deal
|
||||
// with pipes using completely native C++. This is a workaround
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#elif defined(_LIBCPP_VERSION) // libc++
|
||||
template<class CharType, class Traits = std::char_traits<CharType> >
|
||||
class basic_pipebuf : public basic_filebuf<CharType, Traits> {
|
||||
public:
|
||||
typedef basic_pipebuf<CharType, Traits> ThisType;
|
||||
|
||||
public:
|
||||
basic_pipebuf(FILE *fptr, std::ios_base::openmode mode)
|
||||
: basic_filebuf<CharType, Traits>() {
|
||||
this->open(fptr, mode);
|
||||
if (!this->is_open()) {
|
||||
KALDI_WARN << "Error initializing pipebuf"; // probably indicates
|
||||
// code error, if the fptr was good.
|
||||
return;
|
||||
}
|
||||
}
|
||||
}; // class basic_pipebuf
|
||||
#else
|
||||
template<class CharType, class Traits = std::char_traits<CharType> >
|
||||
class basic_pipebuf : public std::basic_filebuf<CharType, Traits> {
|
||||
public:
|
||||
typedef basic_pipebuf<CharType, Traits> ThisType;
|
||||
|
||||
public:
|
||||
basic_pipebuf(FILE *fptr, std::ios_base::openmode mode)
|
||||
: std::basic_filebuf<CharType, Traits>() {
|
||||
this->_M_file.sys_open(fptr, mode);
|
||||
if (!this->is_open()) {
|
||||
KALDI_WARN << "Error initializing pipebuf"; // probably indicates
|
||||
// code error, if the fptr was good.
|
||||
return;
|
||||
}
|
||||
this->_M_mode = mode;
|
||||
this->_M_buf_size = BUFSIZ;
|
||||
this->_M_allocate_internal_buffer();
|
||||
this->_M_reading = false;
|
||||
this->_M_writing = false;
|
||||
this->_M_set_buffer(-1);
|
||||
}
|
||||
}; // class basic_pipebuf
|
||||
#endif // _MSC_VER
|
||||
|
||||
} // namespace kaldi
|
||||
|
||||
#endif // KALDI_UTIL_KALDI_PIPEBUF_H_
|
||||
@@ -0,0 +1,57 @@
|
||||
// util/kaldi-semaphore.cc
|
||||
|
||||
// Copyright 2012 Karel Vesely (Brno University of Technology)
|
||||
// 2017 Dogan Can (University of Southern California)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
|
||||
#include "base/kaldi-error.h"
|
||||
#include "util/kaldi-semaphore.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
Semaphore::Semaphore(int32 count) {
|
||||
KALDI_ASSERT(count >= 0);
|
||||
count_ = count;
|
||||
}
|
||||
|
||||
Semaphore::~Semaphore() {}
|
||||
|
||||
bool Semaphore::TryWait() {
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
if(count_) {
|
||||
count_--;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void Semaphore::Wait() {
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
while(!count_)
|
||||
condition_variable_.wait(lock);
|
||||
count_--;
|
||||
}
|
||||
|
||||
void Semaphore::Signal() {
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
count_++;
|
||||
condition_variable_.notify_one();
|
||||
}
|
||||
|
||||
} // namespace kaldi
|
||||
@@ -0,0 +1,50 @@
|
||||
// util/kaldi-semaphore.h
|
||||
|
||||
// Copyright 2012 Karel Vesely (Brno University of Technology)
|
||||
// 2017 Dogan Can (University of Southern California)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#ifndef KALDI_THREAD_KALDI_SEMAPHORE_H_
|
||||
#define KALDI_THREAD_KALDI_SEMAPHORE_H_ 1
|
||||
|
||||
#include <mutex>
|
||||
#include <condition_variable>
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
class Semaphore {
|
||||
public:
|
||||
Semaphore(int32 count = 0);
|
||||
|
||||
~Semaphore();
|
||||
|
||||
bool TryWait(); ///< Returns true if Wait() goes through
|
||||
void Wait(); ///< decrease the counter
|
||||
void Signal(); ///< increase the counter
|
||||
|
||||
private:
|
||||
int32 count_; ///< the semaphore counter, 0 means block on Wait()
|
||||
|
||||
std::mutex mutex_;
|
||||
std::condition_variable condition_variable_;
|
||||
KALDI_DISALLOW_COPY_AND_ASSIGN(Semaphore);
|
||||
};
|
||||
|
||||
} //namespace
|
||||
|
||||
#endif // KALDI_THREAD_KALDI_SEMAPHORE_H_
|
||||
@@ -0,0 +1,133 @@
|
||||
// util/kaldi-thread-test.cc
|
||||
|
||||
// Copyright 2012 Johns Hopkins University (Author: Daniel Povey)
|
||||
// Frantisek Skala
|
||||
// 2017 University of Southern California (Author: Dogan Can)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <algorithm>
|
||||
#include "base/kaldi-common.h"
|
||||
#include "util/kaldi-thread.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
// Sums up integers from 0 to max_to_count-1.
|
||||
class MyThreadClass : public MultiThreadable {
|
||||
public:
|
||||
MyThreadClass(int32 max_to_count, int32 *i):
|
||||
max_to_count_(max_to_count), iptr_(i), private_counter_(0) { }
|
||||
|
||||
// We are defining a copy constructor to ensure that whenever an instance of
|
||||
// this class is copied, the default *copy* constructor for MultiThreadable
|
||||
// is called instead the default constructor for MultiThreadable.
|
||||
MyThreadClass(const MyThreadClass &other):
|
||||
MultiThreadable(other),
|
||||
max_to_count_(other.max_to_count_), iptr_(other.iptr_),
|
||||
private_counter_(0) { }
|
||||
|
||||
void operator() () {
|
||||
int32 block_size = (max_to_count_+ (num_threads_-1) ) / num_threads_;
|
||||
int32 start = block_size * thread_id_,
|
||||
end = std::min(max_to_count_, start + block_size);
|
||||
for (int32 j = start; j < end; j++)
|
||||
private_counter_ += j;
|
||||
}
|
||||
|
||||
~MyThreadClass() {
|
||||
*iptr_ += private_counter_;
|
||||
}
|
||||
|
||||
private:
|
||||
MyThreadClass() { } // Disallow empty constructor.
|
||||
int32 max_to_count_;
|
||||
int32 *iptr_;
|
||||
int32 private_counter_;
|
||||
};
|
||||
|
||||
|
||||
void TestThreads() {
|
||||
g_num_threads = 8;
|
||||
// run method with temporary threads on 8 threads
|
||||
// Note: uncomment following line for the possibility of simple benchmarking
|
||||
// for(int i=0; i<100000; i++)
|
||||
{
|
||||
int32 max_to_count = 10000, tot = 0;
|
||||
MyThreadClass c(max_to_count, &tot);
|
||||
RunMultiThreaded(c);
|
||||
KALDI_ASSERT(tot == (10000*(10000-1))/2);
|
||||
}
|
||||
g_num_threads = 1;
|
||||
// let's try the same, but with only one thread
|
||||
{
|
||||
int32 max_to_count = 10000, tot = 0;
|
||||
MyThreadClass c(max_to_count, &tot);
|
||||
RunMultiThreaded(c);
|
||||
KALDI_ASSERT(tot == (10000*(10000-1))/2);
|
||||
}
|
||||
}
|
||||
|
||||
class MyTaskClass { // spins for a while, then outputs a pre-given integer.
|
||||
public:
|
||||
MyTaskClass(int32 i, std::vector<int32> *vec):
|
||||
done_(false), i_(i), vec_(vec) { }
|
||||
|
||||
void operator() () {
|
||||
int32 spin = 1000000 * Rand() % 100;
|
||||
for (int32 i = 0; i < spin; i++);
|
||||
done_ = true;
|
||||
}
|
||||
~MyTaskClass() {
|
||||
KALDI_ASSERT(done_);
|
||||
vec_->push_back(i_);
|
||||
}
|
||||
|
||||
private:
|
||||
bool done_;
|
||||
int32 i_;
|
||||
std::vector<int32> *vec_;
|
||||
};
|
||||
|
||||
|
||||
void TestTaskSequencer() {
|
||||
TaskSequencerConfig config;
|
||||
config.num_threads = 1 + Rand() % 20;
|
||||
if (Rand() % 2 == 1 )
|
||||
config.num_threads_total = config.num_threads + Rand() % config.num_threads;
|
||||
|
||||
int32 num_tasks = Rand() % 100;
|
||||
|
||||
std::vector<int32> task_output;
|
||||
{
|
||||
TaskSequencer<MyTaskClass> sequencer(config);
|
||||
for (int32 i = 0; i < num_tasks; i++) {
|
||||
sequencer.Run(new MyTaskClass(i, &task_output));
|
||||
}
|
||||
} // and let "sequencer" be destroyed, which waits for the last threads.
|
||||
KALDI_ASSERT(task_output.size() == static_cast<size_t>(num_tasks));
|
||||
for (int32 i = 0; i < num_tasks; i++)
|
||||
KALDI_ASSERT(task_output[i] == i);
|
||||
}
|
||||
|
||||
|
||||
} // end namespace kaldi.
|
||||
|
||||
int main() {
|
||||
using namespace kaldi;
|
||||
TestThreads();
|
||||
for (int32 i = 0; i < 10; i++)
|
||||
TestTaskSequencer();
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
// util/kaldi-thread.cc
|
||||
|
||||
// Copyright 2012 Johns Hopkins University (Author: Daniel Povey)
|
||||
// Frantisek Skala
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "base/kaldi-common.h"
|
||||
#include "util/kaldi-thread.h"
|
||||
|
||||
namespace kaldi {
|
||||
int32 g_num_threads = 8; // Initialize this global variable.
|
||||
|
||||
MultiThreadable::~MultiThreadable() {
|
||||
// default implementation does nothing
|
||||
}
|
||||
|
||||
|
||||
|
||||
} // end namespace kaldi
|
||||
@@ -0,0 +1,290 @@
|
||||
// util/kaldi-thread.h
|
||||
|
||||
// Copyright 2012 Johns Hopkins University (Author: Daniel Povey)
|
||||
// Frantisek Skala
|
||||
// 2017 University of Southern California (Author: Dogan Can)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_THREAD_KALDI_THREAD_H_
|
||||
#define KALDI_THREAD_KALDI_THREAD_H_ 1
|
||||
|
||||
#include <thread>
|
||||
#include <algorithm>
|
||||
#include "itf/options-itf.h"
|
||||
#include "util/kaldi-semaphore.h"
|
||||
|
||||
// This header provides convenient mechanisms for parallelization.
|
||||
//
|
||||
// The class MultiThreader, and the function RunMultiThreaded provide a
|
||||
// mechanism to run a specified number of jobs in parellel and wait for them
|
||||
// all to finish. They accept objects of some class C that derives from the
|
||||
// base class MultiThreadable. C needs to define the operator () that takes
|
||||
// no arguments. See ExampleClass below.
|
||||
//
|
||||
// The class TaskSequencer addresses a different problem typically encountered
|
||||
// in Kaldi command-line programs that process a sequence of items. The items
|
||||
// to be processed are coming in. They are all of different sizes, e.g.
|
||||
// utterances with different numbers of frames. We would like them to be
|
||||
// processed in parallel to make good use of the threads available but they
|
||||
// must be output in the same order they came in. Here, we again accept objects
|
||||
// of some class C with an operator () that takes no arguments. C may also have
|
||||
// a destructor with side effects (typically some kind of output).
|
||||
// TaskSequencer is responsible for running the jobs in parallel. It has a
|
||||
// function Run() that will accept a new object of class C; this will block
|
||||
// until a thread is free, at which time it will spawn a thread that starts
|
||||
// running the operator () of the object. When threads are finished running,
|
||||
// the objects will be deleted. TaskSequencer guarantees that the destructors
|
||||
// will be called sequentially (not in parallel) and in the same order the
|
||||
// objects were given to the Run() function, so that it is safe for the
|
||||
// destructor to have side effects such as outputting data.
|
||||
// Note: the destructor of TaskSequencer will wait for any remaining jobs that
|
||||
// are still running and will call the destructors.
|
||||
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
extern int32 g_num_threads; // Maximum number of threads (for programs that
|
||||
// use threads, which is not many of them, e.g. the SGMM update program does.
|
||||
// This is 8 by default. You can change this on the command line, where
|
||||
// used, with --num-threads. Programs that think they will use threads
|
||||
// should register it with their ParseOptions, as something like:
|
||||
// po.Register("num-threads", &g_num_threads, "Number of threads to use.");
|
||||
|
||||
class MultiThreadable {
|
||||
// To create a function object that does part of the job, inherit from this
|
||||
// class, implement a copy constructor calling the default copy constructor
|
||||
// of this base class (so that thread_id_ and num_threads_ are copied to new
|
||||
// instances), and finally implement the operator() that does part of the job
|
||||
// based on thread_id_ and num_threads_ variables.
|
||||
// Note: example implementations are in util/kaldi-thread-test.cc
|
||||
public:
|
||||
virtual void operator() () = 0;
|
||||
// Does the main function of the class
|
||||
// Subclasses have to redefine this
|
||||
virtual ~MultiThreadable();
|
||||
// Optional destructor. Note: the destructor of the object passed by the user
|
||||
// will also be called, so watch out.
|
||||
|
||||
public:
|
||||
// Do not redeclare thread_id_ and num_threads_ in derived classes.
|
||||
int32 thread_id_; // 0 <= thread_id_ < num_threads_
|
||||
int32 num_threads_;
|
||||
|
||||
private:
|
||||
// Have additional member variables as needed.
|
||||
};
|
||||
|
||||
|
||||
class ExampleClass: public MultiThreadable {
|
||||
public:
|
||||
ExampleClass(int32 *foo); // Typically there will be an initializer that
|
||||
// takes arguments.
|
||||
|
||||
ExampleClass(const ExampleClass &other); // A copy constructor is also needed;
|
||||
// some example classes use the default version of this.
|
||||
|
||||
void operator() () {
|
||||
// Does the main function of the class. This
|
||||
// function will typically want to look at the values of the
|
||||
// member variables thread_id_ and num_threads_, inherited
|
||||
// from MultiThreadable.
|
||||
}
|
||||
~ExampleClass() {
|
||||
// Optional destructor. Sometimes useful things happen here,
|
||||
// for example summing up of certain quantities. See code
|
||||
// that uses RunMultiThreaded for examples.
|
||||
}
|
||||
private:
|
||||
// Have additional member variables as needed.
|
||||
};
|
||||
|
||||
|
||||
template<class C>
|
||||
class MultiThreader {
|
||||
public:
|
||||
MultiThreader(int32 num_threads, const C &c_in) :
|
||||
threads_(std::max<int32>(1, num_threads)),
|
||||
cvec_(std::max<int32>(1, num_threads), c_in) {
|
||||
if (num_threads == 0) {
|
||||
// This is a special case with num_threads == 0, which behaves like with
|
||||
// num_threads == 1 but without creating extra threads. This can be
|
||||
// useful in GPU computations where threads cannot be used.
|
||||
cvec_[0].thread_id_ = 0;
|
||||
cvec_[0].num_threads_ = 1;
|
||||
(cvec_[0])();
|
||||
} else {
|
||||
for (int32 i = 0; i < threads_.size(); i++) {
|
||||
cvec_[i].thread_id_ = i;
|
||||
cvec_[i].num_threads_ = threads_.size();
|
||||
threads_[i] = std::thread(std::ref(cvec_[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
~MultiThreader() {
|
||||
for (size_t i = 0; i < threads_.size(); i++)
|
||||
if (threads_[i].joinable())
|
||||
threads_[i].join();
|
||||
}
|
||||
private:
|
||||
std::vector<std::thread> threads_;
|
||||
std::vector<C> cvec_;
|
||||
};
|
||||
|
||||
/// Here, class C should inherit from MultiThreadable. Note: if you want to
|
||||
/// control the number of threads yourself, or need to do something in the main
|
||||
/// thread of the program while the objects exist, just initialize the
|
||||
/// MultiThreader<C> object yourself.
|
||||
template<class C> void RunMultiThreaded(const C &c_in) {
|
||||
MultiThreader<C> m(g_num_threads, c_in);
|
||||
}
|
||||
|
||||
|
||||
struct TaskSequencerConfig {
|
||||
int32 num_threads;
|
||||
int32 num_threads_total;
|
||||
TaskSequencerConfig(): num_threads(1), num_threads_total(0) { }
|
||||
void Register(OptionsItf *opts) {
|
||||
opts->Register("num-threads", &num_threads, "Number of actively processing "
|
||||
"threads to run in parallel");
|
||||
opts->Register("num-threads-total", &num_threads_total, "Total number of "
|
||||
"threads, including those that are waiting on other threads "
|
||||
"to produce their output. Controls memory use. If <= 0, "
|
||||
"defaults to --num-threads plus 20. Otherwise, must "
|
||||
"be >= num-threads.");
|
||||
}
|
||||
};
|
||||
|
||||
// C should have an operator () taking no arguments, that does some kind
|
||||
// of computation, and a destructor that produces some kind of output (the
|
||||
// destructors will be run sequentially in the same order Run as called.
|
||||
template<class C>
|
||||
class TaskSequencer {
|
||||
public:
|
||||
TaskSequencer(const TaskSequencerConfig &config):
|
||||
num_threads_(config.num_threads),
|
||||
threads_avail_(config.num_threads),
|
||||
tot_threads_avail_(config.num_threads_total > 0 ? config.num_threads_total :
|
||||
config.num_threads + 20),
|
||||
thread_list_(NULL) {
|
||||
KALDI_ASSERT((config.num_threads_total <= 0 ||
|
||||
config.num_threads_total >= config.num_threads) &&
|
||||
"num-threads-total, if specified, must be >= num-threads");
|
||||
}
|
||||
|
||||
/// This function takes ownership of the pointer "c", and will delete it
|
||||
/// in the same sequence as Run was called on the jobs.
|
||||
void Run(C *c) {
|
||||
// run in main thread
|
||||
if (num_threads_ == 0) {
|
||||
(*c)();
|
||||
delete c;
|
||||
return;
|
||||
}
|
||||
|
||||
threads_avail_.Wait(); // wait till we have a thread for computation free.
|
||||
tot_threads_avail_.Wait(); // this ensures we don't have too many threads
|
||||
// waiting on I/O, and consume too much memory.
|
||||
|
||||
// put the new RunTaskArgsList object at head of the singly
|
||||
// linked list thread_list_.
|
||||
thread_list_ = new RunTaskArgsList(this, c, thread_list_);
|
||||
thread_list_->thread = std::thread(TaskSequencer<C>::RunTask,
|
||||
thread_list_);
|
||||
}
|
||||
|
||||
void Wait() { // You call this at the end if it's more convenient
|
||||
// than waiting for the destructor. It waits for all tasks to finish.
|
||||
if (thread_list_ != NULL) {
|
||||
while (!thread_list_->thread.joinable()) {
|
||||
Sleep(1);
|
||||
}
|
||||
thread_list_->thread.join();
|
||||
KALDI_ASSERT(thread_list_->tail == NULL); // thread would not
|
||||
// have exited without setting tail to NULL.
|
||||
delete thread_list_;
|
||||
thread_list_ = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/// The destructor waits for the last thread to exit.
|
||||
~TaskSequencer() {
|
||||
Wait();
|
||||
}
|
||||
private:
|
||||
struct RunTaskArgsList {
|
||||
TaskSequencer *me; // Think of this as a "this" pointer.
|
||||
C *c; // Clist element of the task we're expected
|
||||
std::thread thread;
|
||||
RunTaskArgsList *tail;
|
||||
RunTaskArgsList(TaskSequencer *me, C *c, RunTaskArgsList *tail):
|
||||
me(me), c(c), tail(tail) {}
|
||||
};
|
||||
// This static function gets run in the threads that we create.
|
||||
static void RunTask(RunTaskArgsList *args) {
|
||||
// (1) run the job.
|
||||
(*(args->c))(); // call operator () on args->c, which does the computation.
|
||||
args->me->threads_avail_.Signal(); // Signal that the compute-intensive
|
||||
// part of the thread is done (we want to run no more than
|
||||
// config_.num_threads of these.)
|
||||
|
||||
// (2) we want to destroy the object "c" now, by deleting it. But for
|
||||
// correct sequencing (this is the whole point of this class, it
|
||||
// is intended to ensure the output of the program is in correct order),
|
||||
// we first wait till the previous thread, whose details will be in "tail",
|
||||
// is finished.
|
||||
if (args->tail != NULL) {
|
||||
while (!args->tail->thread.joinable()){
|
||||
Sleep(1);
|
||||
}
|
||||
args->tail->thread.join();
|
||||
}
|
||||
delete args->c; // delete the object "c". This may cause some output,
|
||||
// e.g. to a stream. We don't need to worry about concurrent access to
|
||||
// the output stream, because each thread waits for the previous thread
|
||||
// to be done, before doing this. So there is no risk of concurrent
|
||||
// access.
|
||||
args->c = NULL;
|
||||
|
||||
if (args->tail != NULL) {
|
||||
KALDI_ASSERT(args->tail->tail == NULL); // Because we already
|
||||
// did join on args->tail->thread, which means that
|
||||
// thread was done, and before it exited, it would have
|
||||
// deleted and set to NULL its tail (which is the next line of code).
|
||||
delete args->tail;
|
||||
args->tail = NULL;
|
||||
}
|
||||
// At this point we are exiting from the thread. Signal the
|
||||
// "tot_threads_avail_" semaphore which is used to limit the total number of threads that are alive, including
|
||||
// not onlhy those that are in active computation in c->operator (), but those
|
||||
// that are waiting on I/O or other threads.
|
||||
args->me->tot_threads_avail_.Signal();
|
||||
}
|
||||
|
||||
int32 num_threads_; // copy of config.num_threads (since Semaphore doesn't store original count)
|
||||
|
||||
Semaphore threads_avail_; // Initialized to the number of threads we are
|
||||
// supposed to run with; the function Run() waits on this.
|
||||
|
||||
Semaphore tot_threads_avail_; // We use this semaphore to ensure we don't
|
||||
// consume too much memory...
|
||||
RunTaskArgsList *thread_list_;
|
||||
|
||||
};
|
||||
|
||||
} // namespace kaldi
|
||||
|
||||
#endif // KALDI_THREAD_KALDI_THREAD_H_
|
||||
@@ -0,0 +1,319 @@
|
||||
// util/parse-options-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// Copyright 2012-2013 Frantisek Skala; Arnab Ghoshal
|
||||
// Copyright 2013 Tanel Alumae
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "util/parse-options.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
struct DummyOptions {
|
||||
int32 my_int;
|
||||
bool my_bool;
|
||||
std::string my_string;
|
||||
|
||||
DummyOptions() {
|
||||
my_int = 0;
|
||||
my_bool = true;
|
||||
my_string = "default dummy string";
|
||||
}
|
||||
|
||||
void Register(ParseOptions *po) {
|
||||
po->Register("my-int", &my_int,
|
||||
"An int32 variable in DummyOptions.");
|
||||
po->Register("my-bool", &my_bool,
|
||||
"A Boolean varaible in DummyOptions.");
|
||||
po->Register("my-str", &my_string,
|
||||
"A string varaible in DummyOptions.");
|
||||
}
|
||||
};
|
||||
|
||||
void UnitTestParseOptions() {
|
||||
int argc = 7;
|
||||
std::string str="default_for_str";
|
||||
int32 num = 1;
|
||||
uint32 unum = 2;
|
||||
const char *argv[7] = { "program_name", "--unum=5", "--num=3", "--i=boo",
|
||||
"a", "b", "c" };
|
||||
ParseOptions po("my usage msg");
|
||||
po.Register("i", &str, "My variable");
|
||||
po.Register("num", &num, "My int32 variable");
|
||||
po.Register("unum", &unum, "My uint32 variable");
|
||||
po.Read(argc, argv);
|
||||
KALDI_ASSERT(po.NumArgs() == 3);
|
||||
KALDI_ASSERT(po.GetArg(1) == "a");
|
||||
KALDI_ASSERT(po.GetArg(2) == "b");
|
||||
KALDI_ASSERT(po.GetArg(3) == "c");
|
||||
KALDI_ASSERT(unum == 5);
|
||||
KALDI_ASSERT(num == 3);
|
||||
KALDI_ASSERT(str == "boo");
|
||||
|
||||
ParseOptions po2("my another msg");
|
||||
int argc2 = 4;
|
||||
const char *argv2[4] = { "program_name", "--i=foo",
|
||||
"--to-be-NORMALIZED=test", "c" };
|
||||
std::string str2 = "default_for_str2";
|
||||
po2.Register("To_Be_Normalized", &str2,
|
||||
"My variable (name has to be normalized)");
|
||||
po2.Register("i", &str, "My variable");
|
||||
po2.Read(argc2, argv2);
|
||||
KALDI_ASSERT(po2.NumArgs() == 1);
|
||||
KALDI_ASSERT(po2.GetArg(1) == "c");
|
||||
KALDI_ASSERT(str2 == "test");
|
||||
KALDI_ASSERT(str == "foo");
|
||||
|
||||
ParseOptions po3("now checking options with prefix");
|
||||
ParseOptions ro3("prefix", &po3); // to register with prefix
|
||||
ParseOptions so3("prefix2", &ro3); // to register with prefix, recursively.
|
||||
DummyOptions dummy_opts;
|
||||
po3.Register("str", &str, "My string variable");
|
||||
po3.Register("num", &num, "My int32 variable");
|
||||
// Now register with prefix
|
||||
ro3.Register("unum", &unum, "My uint32 variable");
|
||||
ro3.Register("str", &str2, "My other string variable");
|
||||
uint32 unum2 = 0;
|
||||
so3.Register("unum", &unum2, "Another uint32 variable");
|
||||
|
||||
int argc3 = 10;
|
||||
const char *argv3[10] = {
|
||||
"program_name", "--prefix.unum=5", "--num=3",
|
||||
"--prefix.str=foo", "--str=bar", "--prefix.my-bool=false",
|
||||
"--prefix.my-str=baz", "--prefix.prefix2.unum=42", "a", "b" };
|
||||
|
||||
dummy_opts.Register(&ro3);
|
||||
po3.PrintUsage(false);
|
||||
|
||||
po3.Read(argc3, argv3);
|
||||
KALDI_ASSERT(po3.NumArgs() == 2);
|
||||
KALDI_ASSERT(po3.GetArg(1) == "a");
|
||||
KALDI_ASSERT(po3.GetArg(2) == "b");
|
||||
KALDI_ASSERT(unum == 5);
|
||||
KALDI_ASSERT(unum2 == 42);
|
||||
KALDI_ASSERT(num == 3);
|
||||
KALDI_ASSERT(str2 == "foo");
|
||||
KALDI_ASSERT(str == "bar");
|
||||
KALDI_ASSERT(dummy_opts.my_bool == false);
|
||||
KALDI_ASSERT(dummy_opts.my_string == "baz");
|
||||
|
||||
|
||||
try { // test error with --option=, which is not a valid way to set
|
||||
// boolean options.
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--option="};
|
||||
ParseOptions po4("my usage msg");
|
||||
bool val = false;
|
||||
po4.Register("option", &val, "My boolean");
|
||||
po4.Read(argc4, argv4);
|
||||
assert(false); // Should not reach this part of code.
|
||||
} catch(std::exception e) {
|
||||
KALDI_LOG << "Failed to read option (this is expected).";
|
||||
}
|
||||
|
||||
{ // test that --option sets "option" to true, if bool.
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--option"};
|
||||
ParseOptions po4("my usage msg");
|
||||
bool val = false;
|
||||
po4.Register("option", &val, "My boolean");
|
||||
po4.Read(argc4, argv4);
|
||||
KALDI_ASSERT(val == true);
|
||||
}
|
||||
|
||||
|
||||
try { // test error with --option, which is not a valid way to set
|
||||
// string-valued options.
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--option"};
|
||||
ParseOptions po4("my usage msg");
|
||||
std::string val;
|
||||
po4.Register("option", &val, "My string");
|
||||
po4.Read(argc4, argv4);
|
||||
assert(false); // Should not reach this part of code.
|
||||
} catch(std::exception e) {
|
||||
KALDI_LOG << "Failed to read option (this is expected).";
|
||||
}
|
||||
|
||||
{ // test that --option= sets "option" to empty, if string.
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--option="};
|
||||
ParseOptions po4("my usage msg");
|
||||
std::string val = "foo";
|
||||
po4.Register("option", &val, "My boolean");
|
||||
po4.Read(argc4, argv4);
|
||||
KALDI_ASSERT(val.empty());
|
||||
}
|
||||
|
||||
{ // integer options test
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--option=8"};
|
||||
ParseOptions po4("my usage msg");
|
||||
int32 val = 32;
|
||||
po4.Register("option", &val, "My int");
|
||||
po4.Read(argc4, argv4);
|
||||
KALDI_ASSERT(val == 8);
|
||||
}
|
||||
|
||||
{ // float
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--option=8.5"};
|
||||
ParseOptions po4("my usage msg");
|
||||
BaseFloat val = 32.0;
|
||||
po4.Register("option", &val, "My float");
|
||||
po4.Read(argc4, argv4);
|
||||
KALDI_ASSERT(val == 8.5);
|
||||
}
|
||||
{ // string options test
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--option=bar"};
|
||||
ParseOptions po4("my usage msg");
|
||||
std::string val = "foo";
|
||||
po4.Register("option", &val, "My string");
|
||||
po4.Read(argc4, argv4);
|
||||
KALDI_ASSERT(val == "bar");
|
||||
}
|
||||
|
||||
try { // test error with --float=string
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--option=foo"};
|
||||
ParseOptions po4("my usage msg");
|
||||
BaseFloat val = 32.0;
|
||||
po4.Register("option", &val, "My float");
|
||||
po4.Read(argc4, argv4);
|
||||
assert(false); // Should not reach this part of code.
|
||||
} catch(std::exception e) {
|
||||
KALDI_LOG << "Failed to read option (this is expected).";
|
||||
}
|
||||
|
||||
|
||||
try { // test error with --int=string
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--option=foo"};
|
||||
ParseOptions po4("my usage msg");
|
||||
int32 val = 32;
|
||||
po4.Register("option", &val, "My int");
|
||||
po4.Read(argc4, argv4);
|
||||
assert(false); // Should not reach this part of code.
|
||||
} catch(std::exception e) {
|
||||
KALDI_LOG << "Failed to read option (this is expected).";
|
||||
}
|
||||
|
||||
try { // test error with --int=int+garbage
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--option=12xyz"};
|
||||
ParseOptions po4("my usage msg");
|
||||
int32 val = 32;
|
||||
po4.Register("option", &val, "My int");
|
||||
po4.Read(argc4, argv4);
|
||||
assert(false); // Should not reach this part of code.
|
||||
} catch(std::exception e) {
|
||||
KALDI_LOG << "Failed to read option (this is expected).";
|
||||
}
|
||||
|
||||
try { // test error with --unsigned-int=negative-number.
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--option=-13"};
|
||||
ParseOptions po4("my usage msg");
|
||||
uint32 val = 32;
|
||||
po4.Register("option", &val, "My int");
|
||||
po4.Read(argc4, argv4);
|
||||
assert(false); // Should not reach this part of code.
|
||||
} catch(std::exception e) {
|
||||
KALDI_LOG << "Failed to read option (this is expected)xxx.";
|
||||
}
|
||||
|
||||
try { // test error with --bool=string
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--option=foo"};
|
||||
ParseOptions po4("my usage msg");
|
||||
bool val = false;
|
||||
po4.Register("option", &val, "My bool");
|
||||
po4.Read(argc4, argv4);
|
||||
assert(false); // Should not reach this part of code.
|
||||
} catch(std::exception e) {
|
||||
KALDI_LOG << "Failed to read option (this is expected).";
|
||||
}
|
||||
|
||||
|
||||
// test error with --=
|
||||
try {
|
||||
int argc4 = 2;
|
||||
const char *argv4[2] = { "program_name", "--=8"};
|
||||
int32 num = 0;
|
||||
ParseOptions po4("my usage msg");
|
||||
po4.Register("num", &num, "My int32 variable");
|
||||
po4.Read(argc4, argv4);
|
||||
KALDI_ASSERT(num == 0);
|
||||
} catch(std::exception e) {
|
||||
KALDI_LOG << "Failed to read option (this is expected).";
|
||||
}
|
||||
|
||||
// test "--" (no more options)
|
||||
int argc4 = 5;
|
||||
unum = 2;
|
||||
const char *argv4[5] = { "program_name", "--unum=6", "--", "a", "b" };
|
||||
ParseOptions po4("my usage msg");
|
||||
po4.Register("unum", &unum, "My uint32 variable");
|
||||
po4.Read(argc4, argv4);
|
||||
KALDI_ASSERT(po4.NumArgs() == 2);
|
||||
KALDI_ASSERT(po4.GetArg(1) == "a");
|
||||
KALDI_ASSERT(po4.GetArg(2) == "b");
|
||||
KALDI_ASSERT(unum == 6);
|
||||
|
||||
// test obsolete "--" (no more options)
|
||||
int argc5 = 3;
|
||||
unum = 2;
|
||||
const char *argv5[3] = { "program_name", "--unum=7", "--" };
|
||||
ParseOptions po5("my usage msg");
|
||||
po5.Register("unum", &unum, "My uint32 variable");
|
||||
po5.Read(argc5, argv5);
|
||||
KALDI_ASSERT(po5.NumArgs() == 0);
|
||||
KALDI_ASSERT(unum == 7);
|
||||
|
||||
// test that "--foo=bar" after "--" is interpreted as argument
|
||||
int argc6 = 4;
|
||||
unum = 2;
|
||||
const char *argv6[5] = { "program_name", "--unum=8", "--", "--foo=8" };
|
||||
ParseOptions po6("my usage msg");
|
||||
po6.Register("unum", &unum, "My uint32 variable");
|
||||
po6.Read(argc6, argv6);
|
||||
KALDI_ASSERT(po6.NumArgs() == 1);
|
||||
KALDI_ASSERT(po6.GetArg(1) == "--foo=8");
|
||||
|
||||
// test that a second registration is ignored
|
||||
int argc7 = 2;
|
||||
const char *const argv7[] = {"program_name", "--i=8"};
|
||||
ParseOptions po7("my usage msg");
|
||||
|
||||
int i7 = 10, k7 = 20;
|
||||
po7.Register("i", &i7, "My int32 variable");
|
||||
po7.Register("i", &k7, "My int32 variable");
|
||||
|
||||
po7.Read(argc7, argv7);
|
||||
|
||||
KALDI_ASSERT(i7 == 8);
|
||||
KALDI_ASSERT(k7 == 20);
|
||||
}
|
||||
|
||||
|
||||
} // end namespace kaldi.
|
||||
|
||||
int main() {
|
||||
using namespace kaldi;
|
||||
UnitTestParseOptions();
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,669 @@
|
||||
// util/parse-options.cc
|
||||
|
||||
// Copyright 2009-2011 Karel Vesely; Microsoft Corporation;
|
||||
// Saarland University (Author: Arnab Ghoshal);
|
||||
// Copyright 2012-2013 Johns Hopkins University (Author: Daniel Povey);
|
||||
// Frantisek Skala; Arnab Ghoshal
|
||||
// Copyright 2013 Tanel Alumae
|
||||
//
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <iostream>
|
||||
#include <iomanip>
|
||||
#include <fstream>
|
||||
#include <algorithm>
|
||||
#include <cstdlib>
|
||||
#include <cassert>
|
||||
#include <cstring>
|
||||
|
||||
#include "util/parse-options.h"
|
||||
#include "util/text-utils.h"
|
||||
#include "base/kaldi-common.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
|
||||
ParseOptions::ParseOptions(const std::string &prefix,
|
||||
OptionsItf *other):
|
||||
print_args_(false), help_(false), usage_(""), argc_(0), argv_(NULL) {
|
||||
ParseOptions *po = dynamic_cast<ParseOptions*>(other);
|
||||
if (po != NULL && po->other_parser_ != NULL) {
|
||||
// we get here if this constructor is used twice, recursively.
|
||||
other_parser_ = po->other_parser_;
|
||||
} else {
|
||||
other_parser_ = other;
|
||||
}
|
||||
if (po != NULL && po->prefix_ != "") {
|
||||
prefix_ = po->prefix_ + std::string(".") + prefix;
|
||||
} else {
|
||||
prefix_ = prefix;
|
||||
}
|
||||
}
|
||||
|
||||
void ParseOptions::Register(const std::string &name,
|
||||
bool *ptr, const std::string &doc) {
|
||||
RegisterTmpl(name, ptr, doc);
|
||||
}
|
||||
|
||||
void ParseOptions::Register(const std::string &name,
|
||||
int32 *ptr, const std::string &doc) {
|
||||
RegisterTmpl(name, ptr, doc);
|
||||
}
|
||||
|
||||
void ParseOptions::Register(const std::string &name,
|
||||
uint32 *ptr, const std::string &doc) {
|
||||
RegisterTmpl(name, ptr, doc);
|
||||
}
|
||||
|
||||
void ParseOptions::Register(const std::string &name,
|
||||
float *ptr, const std::string &doc) {
|
||||
RegisterTmpl(name, ptr, doc);
|
||||
}
|
||||
|
||||
void ParseOptions::Register(const std::string &name,
|
||||
double *ptr, const std::string &doc) {
|
||||
RegisterTmpl(name, ptr, doc);
|
||||
}
|
||||
|
||||
void ParseOptions::Register(const std::string &name,
|
||||
std::string *ptr, const std::string &doc) {
|
||||
RegisterTmpl(name, ptr, doc);
|
||||
}
|
||||
|
||||
// old-style, used for registering application-specific parameters
|
||||
template<typename T>
|
||||
void ParseOptions::RegisterTmpl(const std::string &name, T *ptr,
|
||||
const std::string &doc) {
|
||||
if (other_parser_ == NULL) {
|
||||
this->RegisterCommon(name, ptr, doc, false);
|
||||
} else {
|
||||
KALDI_ASSERT(prefix_ != "" &&
|
||||
"Cannot use empty prefix when registering with prefix.");
|
||||
std::string new_name = prefix_ + '.' + name; // name becomes prefix.name
|
||||
other_parser_->Register(new_name, ptr, doc);
|
||||
}
|
||||
}
|
||||
|
||||
// does the common part of the job of registering a parameter
|
||||
template<typename T>
|
||||
void ParseOptions::RegisterCommon(const std::string &name, T *ptr,
|
||||
const std::string &doc, bool is_standard) {
|
||||
KALDI_ASSERT(ptr != NULL);
|
||||
std::string idx = name;
|
||||
NormalizeArgName(&idx);
|
||||
if (doc_map_.find(idx) != doc_map_.end())
|
||||
KALDI_WARN << "Registering option twice, ignoring second time: " << name;
|
||||
else
|
||||
this->RegisterSpecific(name, idx, ptr, doc, is_standard);
|
||||
}
|
||||
|
||||
// used to register standard parameters (those that are present in all of the
|
||||
// applications)
|
||||
template<typename T>
|
||||
void ParseOptions::RegisterStandard(const std::string &name, T *ptr,
|
||||
const std::string &doc) {
|
||||
this->RegisterCommon(name, ptr, doc, true);
|
||||
}
|
||||
|
||||
void ParseOptions::RegisterSpecific(const std::string &name,
|
||||
const std::string &idx,
|
||||
bool *b,
|
||||
const std::string &doc,
|
||||
bool is_standard) {
|
||||
bool_map_[idx] = b;
|
||||
doc_map_[idx] = DocInfo(name, doc + " (bool, default = "
|
||||
+ ((*b)? "true)" : "false)"), is_standard);
|
||||
}
|
||||
|
||||
void ParseOptions::RegisterSpecific(const std::string &name,
|
||||
const std::string &idx,
|
||||
int32 *i,
|
||||
const std::string &doc,
|
||||
bool is_standard) {
|
||||
int_map_[idx] = i;
|
||||
std::ostringstream ss;
|
||||
ss << doc << " (int, default = " << *i << ")";
|
||||
doc_map_[idx] = DocInfo(name, ss.str(), is_standard);
|
||||
}
|
||||
|
||||
void ParseOptions::RegisterSpecific(const std::string &name,
|
||||
const std::string &idx,
|
||||
uint32 *u,
|
||||
const std::string &doc,
|
||||
bool is_standard) {
|
||||
uint_map_[idx] = u;
|
||||
std::ostringstream ss;
|
||||
ss << doc << " (uint, default = " << *u << ")";
|
||||
doc_map_[idx] = DocInfo(name, ss.str(), is_standard);
|
||||
}
|
||||
|
||||
void ParseOptions::RegisterSpecific(const std::string &name,
|
||||
const std::string &idx,
|
||||
float *f,
|
||||
const std::string &doc,
|
||||
bool is_standard) {
|
||||
float_map_[idx] = f;
|
||||
std::ostringstream ss;
|
||||
ss << doc << " (float, default = " << *f << ")";
|
||||
doc_map_[idx] = DocInfo(name, ss.str(), is_standard);
|
||||
}
|
||||
|
||||
void ParseOptions::RegisterSpecific(const std::string &name,
|
||||
const std::string &idx,
|
||||
double *f,
|
||||
const std::string &doc,
|
||||
bool is_standard) {
|
||||
double_map_[idx] = f;
|
||||
std::ostringstream ss;
|
||||
ss << doc << " (double, default = " << *f << ")";
|
||||
doc_map_[idx] = DocInfo(name, ss.str(), is_standard);
|
||||
}
|
||||
|
||||
void ParseOptions::RegisterSpecific(const std::string &name,
|
||||
const std::string &idx,
|
||||
std::string *s,
|
||||
const std::string &doc,
|
||||
bool is_standard) {
|
||||
string_map_[idx] = s;
|
||||
doc_map_[idx] = DocInfo(name, doc + " (string, default = \"" + *s + "\")",
|
||||
is_standard);
|
||||
}
|
||||
void ParseOptions::DisableOption(const std::string &name) {
|
||||
if (argv_ != NULL)
|
||||
KALDI_ERR << "DisableOption must not be called after calling Read().";
|
||||
if (doc_map_.erase(name) == 0)
|
||||
KALDI_ERR << "Option " << name
|
||||
<< " was not registered so cannot be disabled: ";
|
||||
bool_map_.erase(name);
|
||||
int_map_.erase(name);
|
||||
uint_map_.erase(name);
|
||||
float_map_.erase(name);
|
||||
double_map_.erase(name);
|
||||
string_map_.erase(name);
|
||||
}
|
||||
|
||||
|
||||
int ParseOptions::NumArgs() const {
|
||||
return positional_args_.size();
|
||||
}
|
||||
|
||||
std::string ParseOptions::GetArg(int i) const {
|
||||
// use KALDI_ERR if code error
|
||||
if (i < 1 || i > static_cast<int>(positional_args_.size()))
|
||||
KALDI_ERR << "ParseOptions::GetArg, invalid index " << i;
|
||||
return positional_args_[i - 1];
|
||||
}
|
||||
|
||||
// We currently do not support any other options.
|
||||
enum ShellType { kBash = 0 };
|
||||
|
||||
// This can be changed in the code if it ever does need to be changed (as it's
|
||||
// unlikely that one compilation of this tool-set would use both shells).
|
||||
static ShellType kShellType = kBash;
|
||||
|
||||
// Returns true if we need to escape a string before putting it into
|
||||
// a shell (mainly thinking of bash shell, but should work for others)
|
||||
// This is for the convenience of the user so command-lines that are
|
||||
// printed out by ParseOptions::Read (with --print-args=true) are
|
||||
// paste-able into the shell and will run. If you use a different type of
|
||||
// shell, it might be necessary to change this function.
|
||||
// But it's mostly a cosmetic issue as it basically affects how
|
||||
// the program echoes its command-line arguments to the screen.
|
||||
static bool MustBeQuoted(const std::string &str, ShellType st) {
|
||||
// Only Bash is supported (for the moment).
|
||||
KALDI_ASSERT(st == kBash && "Invalid shell type.");
|
||||
|
||||
const char *c = str.c_str();
|
||||
if (*c == '\0') {
|
||||
return true; // Must quote empty string
|
||||
} else {
|
||||
const char *ok_chars[2];
|
||||
|
||||
// These seem not to be interpreted as long as there are no other "bad"
|
||||
// characters involved (e.g. "," would be interpreted as part of something
|
||||
// like a{b,c}, but not on its own.
|
||||
ok_chars[kBash] = "[]~#^_-+=:.,/";
|
||||
|
||||
// Just want to make sure that a space character doesn't get automatically
|
||||
// inserted here via an automated style-checking script, like it did before.
|
||||
KALDI_ASSERT(!strchr(ok_chars[kBash], ' '));
|
||||
|
||||
for (; *c != '\0'; c++) {
|
||||
// For non-alphanumeric characters we have a list of characters which
|
||||
// are OK. All others are forbidden (this is easier since the shell
|
||||
// interprets most non-alphanumeric characters).
|
||||
if (!isalnum(*c)) {
|
||||
const char *d;
|
||||
for (d = ok_chars[st]; *d != '\0'; d++) if (*c == *d) break;
|
||||
// If not alphanumeric or one of the "ok_chars", it must be escaped.
|
||||
if (*d == '\0') return true;
|
||||
}
|
||||
}
|
||||
return false; // The string was OK. No quoting or escaping.
|
||||
}
|
||||
}
|
||||
|
||||
// Returns a quoted and escaped version of "str"
|
||||
// which has previously been determined to need escaping.
|
||||
// Our aim is to print out the command line in such a way that if it's
|
||||
// pasted into a shell of ShellType "st" (only bash for now), it
|
||||
// will get passed to the program in the same way.
|
||||
static std::string QuoteAndEscape(const std::string &str, ShellType st) {
|
||||
// Only Bash is supported (for the moment).
|
||||
KALDI_ASSERT(st == kBash && "Invalid shell type.");
|
||||
|
||||
// For now we use the following rules:
|
||||
// In the normal case, we quote with single-quote "'", and to escape
|
||||
// a single-quote we use the string: '\'' (interpreted as closing the
|
||||
// single-quote, putting an escaped single-quote from the shell, and
|
||||
// then reopening the single quote).
|
||||
char quote_char = '\'';
|
||||
const char *escape_str = "'\\''"; // e.g. echo 'a'\''b' returns a'b
|
||||
|
||||
// If the string contains single-quotes that would need escaping this
|
||||
// way, and we determine that the string could be safely double-quoted
|
||||
// without requiring any escaping, then we double-quote the string.
|
||||
// This is the case if the characters "`$\ do not appear in the string.
|
||||
// e.g. see http://www.redhat.com/mirrors/LDP/LDP/abs/html/quotingvar.html
|
||||
const char *c_str = str.c_str();
|
||||
if (strchr(c_str, '\'') && !strpbrk(c_str, "\"`$\\")) {
|
||||
quote_char = '"';
|
||||
escape_str = "\\\""; // should never be accessed.
|
||||
}
|
||||
|
||||
char buf[2];
|
||||
buf[1] = '\0';
|
||||
|
||||
buf[0] = quote_char;
|
||||
std::string ans = buf;
|
||||
const char *c = str.c_str();
|
||||
for (;*c != '\0'; c++) {
|
||||
if (*c == quote_char) {
|
||||
ans += escape_str;
|
||||
} else {
|
||||
buf[0] = *c;
|
||||
ans += buf;
|
||||
}
|
||||
}
|
||||
buf[0] = quote_char;
|
||||
ans += buf;
|
||||
return ans;
|
||||
}
|
||||
|
||||
// static function
|
||||
std::string ParseOptions::Escape(const std::string &str) {
|
||||
return MustBeQuoted(str, kShellType) ? QuoteAndEscape(str, kShellType) : str;
|
||||
}
|
||||
|
||||
|
||||
int ParseOptions::Read(int argc, const char *const argv[]) {
|
||||
argc_ = argc;
|
||||
argv_ = argv;
|
||||
std::string key, value;
|
||||
int i;
|
||||
if (argc > 0) {
|
||||
// set global "const char*" g_program_name (name of the program)
|
||||
// so it can be printed out in error messages;
|
||||
// it's useful because often the stderr of different programs will
|
||||
// be mixed together in the same log file.
|
||||
#ifdef _MSC_VER
|
||||
const char *c = strrchr(argv[0], '\\');
|
||||
#else
|
||||
const char *c = strrchr(argv[0], '/');
|
||||
#endif
|
||||
SetProgramName(c == NULL ? argv[0] : c + 1);
|
||||
}
|
||||
// first pass: look for config parameter, look for priority
|
||||
for (i = 1; i < argc; i++) {
|
||||
if (std::strncmp(argv[i], "--", 2) == 0) {
|
||||
if (std::strcmp(argv[i], "--") == 0) {
|
||||
// a lone "--" marks the end of named options
|
||||
break;
|
||||
}
|
||||
bool has_equal_sign;
|
||||
SplitLongArg(argv[i], &key, &value, &has_equal_sign);
|
||||
NormalizeArgName(&key);
|
||||
Trim(&value);
|
||||
if (key.compare("config") == 0) {
|
||||
ReadConfigFile(value);
|
||||
}
|
||||
if (key.compare("help") == 0) {
|
||||
PrintUsage();
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
bool double_dash_seen = false;
|
||||
// second pass: add the command line options
|
||||
for (i = 1; i < argc; i++) {
|
||||
if (std::strncmp(argv[i], "--", 2) == 0) {
|
||||
if (std::strcmp(argv[i], "--") == 0) {
|
||||
// A lone "--" marks the end of named options.
|
||||
// Skip that option and break the processing of named options
|
||||
i += 1;
|
||||
double_dash_seen = true;
|
||||
break;
|
||||
}
|
||||
bool has_equal_sign;
|
||||
SplitLongArg(argv[i], &key, &value, &has_equal_sign);
|
||||
NormalizeArgName(&key);
|
||||
Trim(&value);
|
||||
if (!SetOption(key, value, has_equal_sign)) {
|
||||
PrintUsage(true);
|
||||
KALDI_ERR << "Invalid option " << argv[i];
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// process remaining arguments as positional
|
||||
for (; i < argc; i++) {
|
||||
if ((std::strcmp(argv[i], "--") == 0) && !double_dash_seen) {
|
||||
double_dash_seen = true;
|
||||
} else {
|
||||
positional_args_.push_back(std::string(argv[i]));
|
||||
}
|
||||
}
|
||||
|
||||
// if the user did not suppress this with --print-args = false....
|
||||
if (print_args_) {
|
||||
std::ostringstream strm;
|
||||
for (int j = 0; j < argc; j++)
|
||||
strm << Escape(argv[j]) << " ";
|
||||
strm << '\n';
|
||||
std::cerr << strm.str() << std::flush;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
|
||||
void ParseOptions::PrintUsage(bool print_command_line) {
|
||||
std::cerr << '\n' << usage_ << '\n';
|
||||
DocMapType::iterator it;
|
||||
// first we print application-specific options
|
||||
bool app_specific_header_printed = false;
|
||||
for (it = doc_map_.begin(); it != doc_map_.end(); ++it) {
|
||||
if (it->second.is_standard_ == false) { // application-specific option
|
||||
if (app_specific_header_printed == false) { // header was not yet printed
|
||||
std::cerr << "Options:" << '\n';
|
||||
app_specific_header_printed = true;
|
||||
}
|
||||
std::cerr << " --" << std::setw(25) << std::left << it->second.name_
|
||||
<< " : " << it->second.use_msg_ << '\n';
|
||||
}
|
||||
}
|
||||
if (app_specific_header_printed == true) {
|
||||
std::cerr << '\n';
|
||||
}
|
||||
|
||||
// then the standard options
|
||||
std::cerr << "Standard options:" << '\n';
|
||||
for (it = doc_map_.begin(); it != doc_map_.end(); ++it) {
|
||||
if (it->second.is_standard_ == true) { // we have standard option
|
||||
std::cerr << " --" << std::setw(25) << std::left << it->second.name_
|
||||
<< " : " << it->second.use_msg_ << '\n';
|
||||
}
|
||||
}
|
||||
std::cerr << '\n';
|
||||
if (print_command_line) {
|
||||
std::ostringstream strm;
|
||||
strm << "Command line was: ";
|
||||
for (int j = 0; j < argc_; j++)
|
||||
strm << Escape(argv_[j]) << " ";
|
||||
strm << '\n';
|
||||
std::cerr << strm.str() << std::flush;
|
||||
}
|
||||
}
|
||||
|
||||
void ParseOptions::PrintConfig(std::ostream &os) {
|
||||
os << '\n' << "[[ Configuration of UI-Registered options ]]" << '\n';
|
||||
std::string key;
|
||||
DocMapType::iterator it;
|
||||
for (it = doc_map_.begin(); it != doc_map_.end(); ++it) {
|
||||
key = it->first;
|
||||
os << it->second.name_ << " = ";
|
||||
if (bool_map_.end() != bool_map_.find(key)) {
|
||||
os << (*bool_map_[key] ? "true" : "false");
|
||||
} else if (int_map_.end() != int_map_.find(key)) {
|
||||
os << (*int_map_[key]);
|
||||
} else if (uint_map_.end() != uint_map_.find(key)) {
|
||||
os << (*uint_map_[key]);
|
||||
} else if (float_map_.end() != float_map_.find(key)) {
|
||||
os << (*float_map_[key]);
|
||||
} else if (double_map_.end() != double_map_.find(key)) {
|
||||
os << (*double_map_[key]);
|
||||
} else if (string_map_.end() != string_map_.find(key)) {
|
||||
os << "'" << *string_map_[key] << "'";
|
||||
} else {
|
||||
KALDI_ERR << "PrintConfig: unrecognized option " << key << "[code error]";
|
||||
}
|
||||
os << '\n';
|
||||
}
|
||||
os << '\n';
|
||||
}
|
||||
|
||||
|
||||
void ParseOptions::ReadConfigFile(const std::string &filename) {
|
||||
std::ifstream is(filename.c_str(), std::ifstream::in);
|
||||
if (!is.good()) {
|
||||
KALDI_ERR << "Cannot open config file: " << filename;
|
||||
}
|
||||
|
||||
std::string line, key, value;
|
||||
int32 line_number = 0;
|
||||
while (std::getline(is, line)) {
|
||||
line_number++;
|
||||
// trim out the comments
|
||||
size_t pos;
|
||||
if ((pos = line.find_first_of('#')) != std::string::npos) {
|
||||
line.erase(pos);
|
||||
}
|
||||
// skip empty lines
|
||||
Trim(&line);
|
||||
if (line.length() == 0) continue;
|
||||
|
||||
if (line.substr(0, 2) != "--") {
|
||||
KALDI_ERR << "Reading config file " << filename
|
||||
<< ": line " << line_number << " does not look like a line "
|
||||
<< "from a Kaldi command-line program's config file: should "
|
||||
<< "be of the form --x=y. Note: config files intended to "
|
||||
<< "be sourced by shell scripts lack the '--'.";
|
||||
}
|
||||
|
||||
// parse option
|
||||
bool has_equal_sign;
|
||||
SplitLongArg(line, &key, &value, &has_equal_sign);
|
||||
NormalizeArgName(&key);
|
||||
Trim(&value);
|
||||
if (!SetOption(key, value, has_equal_sign)) {
|
||||
PrintUsage(true);
|
||||
KALDI_ERR << "Invalid option " << line << " in config file " << filename;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
void ParseOptions::SplitLongArg(const std::string &in,
|
||||
std::string *key,
|
||||
std::string *value,
|
||||
bool *has_equal_sign) {
|
||||
KALDI_ASSERT(in.substr(0, 2) == "--"); // precondition.
|
||||
size_t pos = in.find_first_of('=', 0);
|
||||
if (pos == std::string::npos) { // we allow --option for bools
|
||||
// defaults to empty. We handle this differently in different cases.
|
||||
*key = in.substr(2, in.size()-2); // 2 because starts with --.
|
||||
*value = "";
|
||||
*has_equal_sign = false;
|
||||
} else if (pos == 2) { // we also don't allow empty keys: --=value
|
||||
PrintUsage(true);
|
||||
KALDI_ERR << "Invalid option (no key): " << in;
|
||||
} else { // normal case: --option=value
|
||||
*key = in.substr(2, pos-2); // 2 because starts with --.
|
||||
*value = in.substr(pos + 1);
|
||||
*has_equal_sign = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void ParseOptions::NormalizeArgName(std::string *str) {
|
||||
std::string out;
|
||||
std::string::iterator it;
|
||||
|
||||
for (it = str->begin(); it != str->end(); ++it) {
|
||||
if (*it == '_')
|
||||
out += '-'; // convert _ to -
|
||||
else
|
||||
out += std::tolower(*it);
|
||||
}
|
||||
*str = out;
|
||||
|
||||
KALDI_ASSERT(str->length() > 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
bool ParseOptions::SetOption(const std::string &key,
|
||||
const std::string &value,
|
||||
bool has_equal_sign) {
|
||||
if (bool_map_.end() != bool_map_.find(key)) {
|
||||
if (has_equal_sign && value == "")
|
||||
KALDI_ERR << "Invalid option --" << key << "=";
|
||||
*(bool_map_[key]) = ToBool(value);
|
||||
} else if (int_map_.end() != int_map_.find(key)) {
|
||||
*(int_map_[key]) = ToInt(value);
|
||||
} else if (uint_map_.end() != uint_map_.find(key)) {
|
||||
*(uint_map_[key]) = ToUint(value);
|
||||
} else if (float_map_.end() != float_map_.find(key)) {
|
||||
*(float_map_[key]) = ToFloat(value);
|
||||
} else if (double_map_.end() != double_map_.find(key)) {
|
||||
*(double_map_[key]) = ToDouble(value);
|
||||
} else if (string_map_.end() != string_map_.find(key)) {
|
||||
if (!has_equal_sign)
|
||||
KALDI_ERR << "Invalid option --" << key
|
||||
<< " (option format is --x=y).";
|
||||
*(string_map_[key]) = value;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
bool ParseOptions::ToBool(std::string str) {
|
||||
std::transform(str.begin(), str.end(), str.begin(), ::tolower);
|
||||
|
||||
// allow "" as a valid option for "true", so that --x is the same as --x=true
|
||||
if ((str.compare("true") == 0) || (str.compare("t") == 0)
|
||||
|| (str.compare("1") == 0) || (str.compare("") == 0)) {
|
||||
return true;
|
||||
}
|
||||
if ((str.compare("false") == 0) || (str.compare("f") == 0)
|
||||
|| (str.compare("0") == 0)) {
|
||||
return false;
|
||||
}
|
||||
// if it is neither true nor false:
|
||||
PrintUsage(true);
|
||||
KALDI_ERR << "Invalid format for boolean argument [expected true or false]: "
|
||||
<< str;
|
||||
return false; // never reached
|
||||
}
|
||||
|
||||
|
||||
int32 ParseOptions::ToInt(const std::string &str) {
|
||||
int32 ret;
|
||||
if (!ConvertStringToInteger(str, &ret))
|
||||
KALDI_ERR << "Invalid integer option \"" << str << "\"";
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32 ParseOptions::ToUint(const std::string &str) {
|
||||
uint32 ret;
|
||||
if (!ConvertStringToInteger(str, &ret))
|
||||
KALDI_ERR << "Invalid integer option \"" << str << "\"";
|
||||
return ret;
|
||||
}
|
||||
|
||||
float ParseOptions::ToFloat(const std::string &str) {
|
||||
float ret;
|
||||
if (!ConvertStringToReal(str, &ret))
|
||||
KALDI_ERR << "Invalid floating-point option \"" << str << "\"";
|
||||
return ret;
|
||||
}
|
||||
|
||||
double ParseOptions::ToDouble(const std::string &str) {
|
||||
double ret;
|
||||
if (!ConvertStringToReal(str, &ret))
|
||||
KALDI_ERR << "Invalid floating-point option \"" << str << "\"";
|
||||
return ret;
|
||||
}
|
||||
|
||||
// instantiate templates
|
||||
template void ParseOptions::RegisterTmpl(const std::string &name, bool *ptr,
|
||||
const std::string &doc);
|
||||
template void ParseOptions::RegisterTmpl(const std::string &name, int32 *ptr,
|
||||
const std::string &doc);
|
||||
template void ParseOptions::RegisterTmpl(const std::string &name, uint32 *ptr,
|
||||
const std::string &doc);
|
||||
template void ParseOptions::RegisterTmpl(const std::string &name, float *ptr,
|
||||
const std::string &doc);
|
||||
template void ParseOptions::RegisterTmpl(const std::string &name, double *ptr,
|
||||
const std::string &doc);
|
||||
template void ParseOptions::RegisterTmpl(const std::string &name,
|
||||
std::string *ptr, const std::string &doc);
|
||||
|
||||
template void ParseOptions::RegisterStandard(const std::string &name,
|
||||
bool *ptr,
|
||||
const std::string &doc);
|
||||
template void ParseOptions::RegisterStandard(const std::string &name,
|
||||
int32 *ptr,
|
||||
const std::string &doc);
|
||||
template void ParseOptions::RegisterStandard(const std::string &name,
|
||||
uint32 *ptr,
|
||||
const std::string &doc);
|
||||
template void ParseOptions::RegisterStandard(const std::string &name,
|
||||
float *ptr,
|
||||
const std::string &doc);
|
||||
template void ParseOptions::RegisterStandard(const std::string &name,
|
||||
double *ptr,
|
||||
const std::string &doc);
|
||||
template void ParseOptions::RegisterStandard(const std::string &name,
|
||||
std::string *ptr,
|
||||
const std::string &doc);
|
||||
|
||||
template void ParseOptions::RegisterCommon(const std::string &name,
|
||||
bool *ptr,
|
||||
const std::string &doc, bool is_standard);
|
||||
template void ParseOptions::RegisterCommon(const std::string &name,
|
||||
int32 *ptr,
|
||||
const std::string &doc, bool is_standard);
|
||||
template void ParseOptions::RegisterCommon(const std::string &name,
|
||||
uint32 *ptr,
|
||||
const std::string &doc, bool is_standard);
|
||||
template void ParseOptions::RegisterCommon(const std::string &name,
|
||||
float *ptr,
|
||||
const std::string &doc, bool is_standard);
|
||||
template void ParseOptions::RegisterCommon(const std::string &name,
|
||||
double *ptr,
|
||||
const std::string &doc, bool is_standard);
|
||||
template void ParseOptions::RegisterCommon(const std::string &name,
|
||||
std::string *ptr,
|
||||
const std::string &doc, bool is_standard);
|
||||
|
||||
} // namespace kaldi
|
||||
@@ -0,0 +1,264 @@
|
||||
// util/parse-options.h
|
||||
|
||||
// Copyright 2009-2011 Karel Vesely; Microsoft Corporation;
|
||||
// Saarland University (Author: Arnab Ghoshal);
|
||||
// Copyright 2012-2013 Frantisek Skala; Arnab Ghoshal
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_UTIL_PARSE_OPTIONS_H_
|
||||
#define KALDI_UTIL_PARSE_OPTIONS_H_
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "base/kaldi-common.h"
|
||||
#include "itf/options-itf.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
/// The class ParseOptions is for parsing command-line options; see
|
||||
/// \ref parse_options for more documentation.
|
||||
class ParseOptions : public OptionsItf {
|
||||
public:
|
||||
explicit ParseOptions(const char *usage) :
|
||||
print_args_(true), help_(false), usage_(usage), argc_(0), argv_(NULL),
|
||||
prefix_(""), other_parser_(NULL) {
|
||||
#if !defined(_MSC_VER) && !defined(__CYGWIN__) // This is just a convenient place to set the stderr to line
|
||||
setlinebuf(stderr); // buffering mode, since it's called at program start.
|
||||
#endif // This helps ensure different programs' output is not mixed up.
|
||||
RegisterStandard("config", &config_, "Configuration file to read (this "
|
||||
"option may be repeated)");
|
||||
RegisterStandard("print-args", &print_args_,
|
||||
"Print the command line arguments (to stderr)");
|
||||
RegisterStandard("help", &help_, "Print out usage message");
|
||||
RegisterStandard("verbose", &g_kaldi_verbose_level,
|
||||
"Verbose level (higher->more logging)");
|
||||
}
|
||||
|
||||
/**
|
||||
This is a constructor for the special case where some options are
|
||||
registered with a prefix to avoid conflicts. The object thus created will
|
||||
only be used temporarily to register an options class with the original
|
||||
options parser (which is passed as the *other pointer) using the given
|
||||
prefix. It should not be used for any other purpose, and the prefix must
|
||||
not be the empty string. It seems to be the least bad way of implementing
|
||||
options with prefixes at this point.
|
||||
Example of usage is:
|
||||
ParseOptions po; // original ParseOptions object
|
||||
ParseOptions po_mfcc("mfcc", &po); // object with prefix.
|
||||
MfccOptions mfcc_opts;
|
||||
mfcc_opts.Register(&po_mfcc);
|
||||
The options will now get registered as, e.g., --mfcc.frame-shift=10.0
|
||||
instead of just --frame-shift=10.0
|
||||
*/
|
||||
ParseOptions(const std::string &prefix, OptionsItf *other);
|
||||
|
||||
~ParseOptions() {}
|
||||
|
||||
// Methods from the interface
|
||||
void Register(const std::string &name,
|
||||
bool *ptr, const std::string &doc);
|
||||
void Register(const std::string &name,
|
||||
int32 *ptr, const std::string &doc);
|
||||
void Register(const std::string &name,
|
||||
uint32 *ptr, const std::string &doc);
|
||||
void Register(const std::string &name,
|
||||
float *ptr, const std::string &doc);
|
||||
void Register(const std::string &name,
|
||||
double *ptr, const std::string &doc);
|
||||
void Register(const std::string &name,
|
||||
std::string *ptr, const std::string &doc);
|
||||
|
||||
/// If called after registering an option and before calling
|
||||
/// Read(), disables that option from being used. Will crash
|
||||
/// at runtime if that option had not been registered.
|
||||
void DisableOption(const std::string &name);
|
||||
|
||||
/// This one is used for registering standard parameters of all the programs
|
||||
template<typename T>
|
||||
void RegisterStandard(const std::string &name,
|
||||
T *ptr, const std::string &doc);
|
||||
|
||||
/**
|
||||
Parses the command line options and fills the ParseOptions-registered
|
||||
variables. This must be called after all the variables were registered!!!
|
||||
|
||||
Initially the variables have implicit values,
|
||||
then the config file values are set-up,
|
||||
finally the command line values given.
|
||||
Returns the first position in argv that was not used.
|
||||
[typically not useful: use NumParams() and GetParam(). ]
|
||||
*/
|
||||
int Read(int argc, const char *const *argv);
|
||||
|
||||
/// Prints the usage documentation [provided in the constructor].
|
||||
void PrintUsage(bool print_command_line = false);
|
||||
/// Prints the actual configuration of all the registered variables
|
||||
void PrintConfig(std::ostream &os);
|
||||
|
||||
/// Reads the options values from a config file. Must be called after
|
||||
/// registering all options. This is usually used internally after the
|
||||
/// standard --config option is used, but it may also be called from a
|
||||
/// program.
|
||||
void ReadConfigFile(const std::string &filename);
|
||||
|
||||
/// Number of positional parameters (c.f. argc-1).
|
||||
int NumArgs() const;
|
||||
|
||||
/// Returns one of the positional parameters; 1-based indexing for argc/argv
|
||||
/// compatibility. Will crash if param is not >=1 and <=NumArgs().
|
||||
std::string GetArg(int param) const;
|
||||
|
||||
std::string GetOptArg(int param) const {
|
||||
return (param <= NumArgs() ? GetArg(param) : "");
|
||||
}
|
||||
|
||||
/// The following function will return a possibly quoted and escaped
|
||||
/// version of "str", according to the current shell. Currently
|
||||
/// this is just hardwired to bash. It's useful for debug output.
|
||||
static std::string Escape(const std::string &str);
|
||||
|
||||
private:
|
||||
/// Template to register various variable types,
|
||||
/// used for program-specific parameters
|
||||
template<typename T>
|
||||
void RegisterTmpl(const std::string &name, T *ptr, const std::string &doc);
|
||||
|
||||
// Following functions do just the datatype-specific part of the job
|
||||
/// Register boolean variable
|
||||
void RegisterSpecific(const std::string &name, const std::string &idx,
|
||||
bool *b, const std::string &doc, bool is_standard);
|
||||
/// Register int32 variable
|
||||
void RegisterSpecific(const std::string &name, const std::string &idx,
|
||||
int32 *i, const std::string &doc, bool is_standard);
|
||||
/// Register unsinged int32 variable
|
||||
void RegisterSpecific(const std::string &name, const std::string &idx,
|
||||
uint32 *u,
|
||||
const std::string &doc, bool is_standard);
|
||||
/// Register float variable
|
||||
void RegisterSpecific(const std::string &name, const std::string &idx,
|
||||
float *f, const std::string &doc, bool is_standard);
|
||||
/// Register double variable [useful as we change BaseFloat type].
|
||||
void RegisterSpecific(const std::string &name, const std::string &idx,
|
||||
double *f, const std::string &doc, bool is_standard);
|
||||
/// Register string variable
|
||||
void RegisterSpecific(const std::string &name, const std::string &idx,
|
||||
std::string *s, const std::string &doc,
|
||||
bool is_standard);
|
||||
|
||||
/// Does the actual job for both kinds of parameters
|
||||
/// Does the common part of the job for all datatypes,
|
||||
/// then calls RegisterSpecific
|
||||
template<typename T>
|
||||
void RegisterCommon(const std::string &name,
|
||||
T *ptr, const std::string &doc, bool is_standard);
|
||||
|
||||
/// Set option with name "key" to "value"; will crash if can't do it.
|
||||
/// "has_equal_sign" is used to allow --x for a boolean option x,
|
||||
/// and --y=, for a string option y.
|
||||
bool SetOption(const std::string &key, const std::string &value,
|
||||
bool has_equal_sign);
|
||||
|
||||
bool ToBool(std::string str);
|
||||
int32 ToInt(const std::string &str);
|
||||
uint32 ToUint(const std::string &str);
|
||||
float ToFloat(const std::string &str);
|
||||
double ToDouble(const std::string &str);
|
||||
|
||||
// maps for option variables
|
||||
std::map<std::string, bool*> bool_map_;
|
||||
std::map<std::string, int32*> int_map_;
|
||||
std::map<std::string, uint32*> uint_map_;
|
||||
std::map<std::string, float*> float_map_;
|
||||
std::map<std::string, double*> double_map_;
|
||||
std::map<std::string, std::string*> string_map_;
|
||||
|
||||
/**
|
||||
Structure for options' documentation
|
||||
*/
|
||||
struct DocInfo {
|
||||
DocInfo() {}
|
||||
DocInfo(const std::string &name, const std::string &usemsg)
|
||||
: name_(name), use_msg_(usemsg), is_standard_(false) {}
|
||||
DocInfo(const std::string &name, const std::string &usemsg,
|
||||
bool is_standard)
|
||||
: name_(name), use_msg_(usemsg), is_standard_(is_standard) {}
|
||||
|
||||
std::string name_;
|
||||
std::string use_msg_;
|
||||
bool is_standard_;
|
||||
};
|
||||
typedef std::map<std::string, DocInfo> DocMapType;
|
||||
DocMapType doc_map_; ///< map for the documentation
|
||||
|
||||
bool print_args_; ///< variable for the implicit --print-args parameter
|
||||
bool help_; ///< variable for the implicit --help parameter
|
||||
std::string config_; ///< variable for the implicit --config parameter
|
||||
std::vector<std::string> positional_args_;
|
||||
const char *usage_;
|
||||
int argc_;
|
||||
const char *const *argv_;
|
||||
|
||||
/// These members are not normally used. They are only used when the object
|
||||
/// is constructed with a prefix
|
||||
std::string prefix_;
|
||||
OptionsItf *other_parser_;
|
||||
protected:
|
||||
/// SplitLongArg parses an argument of the form --a=b, --a=, or --a,
|
||||
/// and sets "has_equal_sign" to true if an equals-sign was parsed..
|
||||
/// this is needed in order to correctly allow --x for a boolean option
|
||||
/// x, and --y= for a string option y, and to disallow --x= and --y.
|
||||
void SplitLongArg(const std::string &in, std::string *key,
|
||||
std::string *value, bool *has_equal_sign);
|
||||
|
||||
void NormalizeArgName(std::string *str);
|
||||
};
|
||||
|
||||
/// This template is provided for convenience in reading config classes from
|
||||
/// files; this is not the standard way to read configuration options, but may
|
||||
/// occasionally be needed. This function assumes the config has a function
|
||||
/// "void Register(OptionsItf *opts)" which it can call to register the
|
||||
/// ParseOptions object.
|
||||
template<class C> void ReadConfigFromFile(const std::string &config_filename,
|
||||
C *c) {
|
||||
std::ostringstream usage_str;
|
||||
usage_str << "Parsing config from "
|
||||
<< "from '" << config_filename << "'";
|
||||
ParseOptions po(usage_str.str().c_str());
|
||||
c->Register(&po);
|
||||
po.ReadConfigFile(config_filename);
|
||||
}
|
||||
|
||||
/// This variant of the template ReadConfigFromFile is for if you need to read
|
||||
/// two config classes from the same file.
|
||||
template<class C1, class C2> void ReadConfigsFromFile(const std::string &conf,
|
||||
C1 *c1, C2 *c2) {
|
||||
std::ostringstream usage_str;
|
||||
usage_str << "Parsing config from "
|
||||
<< "from '" << conf << "'";
|
||||
ParseOptions po(usage_str.str().c_str());
|
||||
c1->Register(&po);
|
||||
c2->Register(&po);
|
||||
po.ReadConfigFile(conf);
|
||||
}
|
||||
|
||||
|
||||
|
||||
} // namespace kaldi
|
||||
|
||||
#endif // KALDI_UTIL_PARSE_OPTIONS_H_
|
||||
@@ -0,0 +1,81 @@
|
||||
// util/simple-io-funcs.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "util/simple-io-funcs.h"
|
||||
#include "util/text-utils.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
bool WriteIntegerVectorSimple(const std::string &wxfilename,
|
||||
const std::vector<int32> &list) {
|
||||
kaldi::Output ko;
|
||||
// false, false is: text-mode, no Kaldi header.
|
||||
if (!ko.Open(wxfilename, false, false)) return false;
|
||||
for (size_t i = 0; i < list.size(); i++) ko.Stream() << list[i] << '\n';
|
||||
return ko.Close();
|
||||
}
|
||||
|
||||
bool ReadIntegerVectorSimple(const std::string &rxfilename,
|
||||
std::vector<int32> *list) {
|
||||
kaldi::Input ki;
|
||||
if (!ki.OpenTextMode(rxfilename)) return false;
|
||||
std::istream &is = ki.Stream();
|
||||
int32 i;
|
||||
list->clear();
|
||||
while ( !(is >> i).fail() )
|
||||
list->push_back(i);
|
||||
is >> std::ws;
|
||||
return is.eof(); // should be eof, or junk at end of file.
|
||||
}
|
||||
|
||||
bool WriteIntegerVectorVectorSimple(const std::string &wxfilename,
|
||||
const std::vector<std::vector<int32> > &list) {
|
||||
kaldi::Output ko;
|
||||
// false, false is: text-mode, no Kaldi header.
|
||||
if (!ko.Open(wxfilename, false, false)) return false;
|
||||
std::ostream &os = ko.Stream();
|
||||
for (size_t i = 0; i < list.size(); i++) {
|
||||
for (size_t j = 0; j < list[i].size(); j++) {
|
||||
os << list[i][j];
|
||||
if (j+1 < list[i].size()) os << ' ';
|
||||
}
|
||||
os << '\n';
|
||||
}
|
||||
return ko.Close();
|
||||
}
|
||||
|
||||
bool ReadIntegerVectorVectorSimple(const std::string &rxfilename,
|
||||
std::vector<std::vector<int32> > *list) {
|
||||
kaldi::Input ki;
|
||||
if (!ki.OpenTextMode(rxfilename)) return false;
|
||||
std::istream &is = ki.Stream();
|
||||
list->clear();
|
||||
std::string line;
|
||||
while (std::getline(is, line)) {
|
||||
std::vector<int32> v;
|
||||
if (!SplitStringToIntegers(line, " \t\r", true, &v)) {
|
||||
list->clear();
|
||||
return false;
|
||||
}
|
||||
list->push_back(v);
|
||||
}
|
||||
return is.eof(); // if we're not at EOF, something weird happened.
|
||||
}
|
||||
|
||||
|
||||
} // end namespace kaldi
|
||||
@@ -0,0 +1,63 @@
|
||||
// util/simple-io-funcs.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation; Jan Silovsky
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#ifndef KALDI_UTIL_SIMPLE_IO_FUNCS_H_
|
||||
#define KALDI_UTIL_SIMPLE_IO_FUNCS_H_
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "util/kaldi-io.h"
|
||||
|
||||
// This header contains some utilities for reading some common, simple text
|
||||
// formats:integers in files, one per line, and integers in files, possibly
|
||||
// multiple per line. these are not really fully native Kaldi formats; they are
|
||||
// mostly for small files that might be generated by scripts, and can be read
|
||||
// all at one time. for longer files of this type, we would probably use the
|
||||
// Table code.
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
/// WriteToList attempts to write this list of integers, one per line,
|
||||
/// to the given file, in text format.
|
||||
/// returns true if succeeded.
|
||||
bool WriteIntegerVectorSimple(const std::string &wxfilename,
|
||||
const std::vector<int32> &v);
|
||||
|
||||
/// ReadFromList attempts to read this list of integers, one per line,
|
||||
/// from the given file, in text format.
|
||||
/// returns true if succeeded.
|
||||
bool ReadIntegerVectorSimple(const std::string &rxfilename,
|
||||
std::vector<int32> *v);
|
||||
|
||||
// This is a file format like:
|
||||
// 1 2
|
||||
// 3
|
||||
//
|
||||
// 4 5 6
|
||||
// etc.
|
||||
bool WriteIntegerVectorVectorSimple(const std::string &wxfilename,
|
||||
const std::vector<std::vector<int32> > &v);
|
||||
|
||||
bool ReadIntegerVectorVectorSimple(const std::string &rxfilename,
|
||||
std::vector<std::vector<int32> > *v);
|
||||
|
||||
|
||||
} // end namespace kaldi.
|
||||
|
||||
|
||||
#endif // KALDI_UTIL_SIMPLE_IO_FUNCS_H_
|
||||
@@ -0,0 +1,86 @@
|
||||
// util/simple-options-test.cc
|
||||
|
||||
// Copyright 2013 Tanel Alumae, Tallinn University of Technology
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "util/simple-options.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
|
||||
void UnitTestSimpleOptions() {
|
||||
std::string str="default_for_str";
|
||||
int32 num = 1;
|
||||
uint32 unum = 2;
|
||||
float realnum = 0.1;
|
||||
bool flag = false;
|
||||
bool rval;
|
||||
SimpleOptions so;
|
||||
|
||||
so.Register("num", &num, "Description of num");
|
||||
so.Register("unum", &unum, "Description of unum");
|
||||
so.Register("str", &str, "Description of str");
|
||||
so.Register("flag", &flag, "Description of flag");
|
||||
so.Register("realnum", &realnum, "Description of realnum");
|
||||
|
||||
rval = so.SetOption("num", 42);
|
||||
KALDI_ASSERT(rval);
|
||||
so.SetOption("unum", static_cast<uint32>(43));
|
||||
KALDI_ASSERT(rval);
|
||||
rval = so.SetOption("str", (std::string)"foo");
|
||||
KALDI_ASSERT(rval);
|
||||
rval = so.SetOption("flag", false);
|
||||
KALDI_ASSERT(rval);
|
||||
|
||||
KALDI_ASSERT(num == 42);
|
||||
KALDI_ASSERT(unum == 43);
|
||||
KALDI_ASSERT(str == "foo");
|
||||
KALDI_ASSERT(flag == false);
|
||||
|
||||
rval = so.SetOption("str", "foo2");
|
||||
KALDI_ASSERT(rval);
|
||||
KALDI_ASSERT(str == "foo2");
|
||||
|
||||
// test automatic conversion between int and uint
|
||||
rval = so.SetOption("unum", 44);
|
||||
KALDI_ASSERT(rval);
|
||||
KALDI_ASSERT(unum == 44);
|
||||
|
||||
// test automatic conversion between float and double
|
||||
rval = so.SetOption("realnum", static_cast<float>(0.2));
|
||||
KALDI_ASSERT(rval);
|
||||
KALDI_ASSERT(realnum - 0.2 < 0.000001);
|
||||
rval = so.SetOption("realnum", static_cast<double>(0.3));
|
||||
KALDI_ASSERT(rval);
|
||||
KALDI_ASSERT(realnum - 0.3 < 0.000001);
|
||||
|
||||
SimpleOptions::OptionType type;
|
||||
rval = so.GetOptionType("num", &type);
|
||||
KALDI_ASSERT(rval);
|
||||
KALDI_ASSERT(type == SimpleOptions::kInt32);
|
||||
|
||||
rval = so.GetOptionType("xxxx", &type);
|
||||
KALDI_ASSERT(rval == false);
|
||||
}
|
||||
|
||||
|
||||
} // end namespace kaldi.
|
||||
|
||||
int main() {
|
||||
using namespace kaldi;
|
||||
UnitTestSimpleOptions();
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
// util/simple-options.cc
|
||||
|
||||
// Copyright 2013 Tanel Alumae, Tallinn University of Technology
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#include "util/simple-options.h"
|
||||
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
void SimpleOptions::Register(const std::string &name,
|
||||
bool *value,
|
||||
const std::string &doc) {
|
||||
bool_map_[name] = value;
|
||||
option_info_list_.push_back(std::make_pair(name, OptionInfo(doc, kBool)));
|
||||
}
|
||||
|
||||
void SimpleOptions::Register(const std::string &name,
|
||||
int32 *value,
|
||||
const std::string &doc) {
|
||||
int_map_[name] = value;
|
||||
option_info_list_.push_back(std::make_pair(name, OptionInfo(doc, kInt32)));
|
||||
}
|
||||
|
||||
void SimpleOptions::Register(const std::string &name,
|
||||
uint32 *value,
|
||||
const std::string &doc) {
|
||||
uint_map_[name] = value;
|
||||
option_info_list_.push_back(std::make_pair(name, OptionInfo(doc, kUint32)));
|
||||
}
|
||||
|
||||
void SimpleOptions::Register(const std::string &name,
|
||||
float *value,
|
||||
const std::string &doc) {
|
||||
float_map_[name] = value;
|
||||
option_info_list_.push_back(std::make_pair(name, OptionInfo(doc, kFloat)));
|
||||
}
|
||||
|
||||
void SimpleOptions::Register(const std::string &name,
|
||||
double *value,
|
||||
const std::string &doc) {
|
||||
double_map_[name] = value;
|
||||
option_info_list_.push_back(std::make_pair(name, OptionInfo(doc, kDouble)));
|
||||
}
|
||||
|
||||
void SimpleOptions::Register(const std::string &name,
|
||||
std::string *value,
|
||||
const std::string &doc) {
|
||||
string_map_[name] = value;
|
||||
option_info_list_.push_back(std::make_pair(name, OptionInfo(doc, kString)));
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static bool SetOptionImpl(const std::string &key, const T &value,
|
||||
std::map<std::string, T*> &some_map) {
|
||||
if (some_map.end() != some_map.find(key)) {
|
||||
*(some_map[key]) = value;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool SimpleOptions::SetOption(const std::string &key, const bool &value) {
|
||||
return SetOptionImpl(key, value, bool_map_);
|
||||
}
|
||||
|
||||
bool SimpleOptions::SetOption(const std::string &key, const int32 &value) {
|
||||
if (!SetOptionImpl(key, value, int_map_)) {
|
||||
if (!SetOptionImpl(key, static_cast<uint32>(value), uint_map_)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SimpleOptions::SetOption(const std::string &key, const uint32 &value) {
|
||||
if (!SetOptionImpl(key, value, uint_map_)) {
|
||||
if (!SetOptionImpl(key, static_cast<int32>(value), int_map_)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SimpleOptions::SetOption(const std::string &key, const float &value) {
|
||||
if (!SetOptionImpl(key, value, float_map_)) {
|
||||
if (!SetOptionImpl(key, static_cast<double>(value), double_map_)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SimpleOptions::SetOption(const std::string &key, const double &value) {
|
||||
if (!SetOptionImpl(key, value, double_map_)) {
|
||||
if (!SetOptionImpl(key, static_cast<float>(value), float_map_)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SimpleOptions::SetOption(const std::string &key,
|
||||
const std::string &value) {
|
||||
return SetOptionImpl(key, value, string_map_);
|
||||
}
|
||||
|
||||
bool SimpleOptions::SetOption(const std::string &key, const char *value) {
|
||||
std::string str_value = std::string(value);
|
||||
return SetOptionImpl(key, str_value, string_map_);
|
||||
}
|
||||
|
||||
|
||||
template<typename T>
|
||||
static bool GetOptionImpl(const std::string &key, T *value,
|
||||
std::map<std::string, T*> &some_map) {
|
||||
typename std::map<std::string, T*>::iterator it = some_map.find(key);
|
||||
if (it != some_map.end()) {
|
||||
*value = *(it->second);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool SimpleOptions::GetOption(const std::string &key, bool *value) {
|
||||
return GetOptionImpl(key, value, bool_map_);
|
||||
}
|
||||
|
||||
bool SimpleOptions::GetOption(const std::string &key, int32 *value) {
|
||||
return GetOptionImpl(key, value, int_map_);
|
||||
}
|
||||
|
||||
bool SimpleOptions::GetOption(const std::string &key, uint32 *value) {
|
||||
return GetOptionImpl(key, value, uint_map_);
|
||||
}
|
||||
|
||||
bool SimpleOptions::GetOption(const std::string &key, float *value) {
|
||||
return GetOptionImpl(key, value, float_map_);
|
||||
}
|
||||
|
||||
bool SimpleOptions::GetOption(const std::string &key, double *value) {
|
||||
return GetOptionImpl(key, value, double_map_);
|
||||
}
|
||||
|
||||
bool SimpleOptions::GetOption(const std::string &key, std::string *value) {
|
||||
return GetOptionImpl(key, value, string_map_);
|
||||
}
|
||||
|
||||
std::vector<std::pair<std::string, SimpleOptions::OptionInfo> >
|
||||
SimpleOptions::GetOptionInfoList() {
|
||||
return option_info_list_;
|
||||
}
|
||||
|
||||
bool SimpleOptions::GetOptionType(const std::string &key, OptionType *type) {
|
||||
for (std::vector <std::pair<std::string,
|
||||
OptionInfo> >::iterator dx = option_info_list_.begin();
|
||||
dx != option_info_list_.end(); dx++) {
|
||||
std::pair<std::string, SimpleOptions::OptionInfo> info_pair = (*dx);
|
||||
if (info_pair.first == key) {
|
||||
*type = info_pair.second.type;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
||||
} // namespace kaldi
|
||||
@@ -0,0 +1,113 @@
|
||||
// util/simple-options.h
|
||||
|
||||
// Copyright 2013 Tanel Alumae, Tallinn University of Technology
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_UTIL_SIMPLE_OPTIONS_H_
|
||||
#define KALDI_UTIL_SIMPLE_OPTIONS_H_
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "base/kaldi-common.h"
|
||||
#include "itf/options-itf.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
|
||||
/// The class SimpleOptions is an implementation of OptionsItf that allows
|
||||
/// setting and getting option values programmatically, i.e., via getter
|
||||
/// and setter methods. It doesn't provide any command line parsing
|
||||
/// functionality.
|
||||
/// The class ParseOptions should be used for command-line options.
|
||||
class SimpleOptions : public OptionsItf {
|
||||
public:
|
||||
SimpleOptions() {
|
||||
}
|
||||
|
||||
virtual ~SimpleOptions() {
|
||||
}
|
||||
|
||||
// Methods from the interface
|
||||
void Register(const std::string &name, bool *ptr, const std::string &doc);
|
||||
void Register(const std::string &name, int32 *ptr, const std::string &doc);
|
||||
void Register(const std::string &name, uint32 *ptr, const std::string &doc);
|
||||
void Register(const std::string &name, float *ptr, const std::string &doc);
|
||||
void Register(const std::string &name, double *ptr, const std::string &doc);
|
||||
void Register(const std::string &name, std::string *ptr,
|
||||
const std::string &doc);
|
||||
|
||||
// set option with the specified key, return true if successful
|
||||
bool SetOption(const std::string &key, const bool &value);
|
||||
bool SetOption(const std::string &key, const int32 &value);
|
||||
bool SetOption(const std::string &key, const uint32 &value);
|
||||
bool SetOption(const std::string &key, const float &value);
|
||||
bool SetOption(const std::string &key, const double &value);
|
||||
bool SetOption(const std::string &key, const std::string &value);
|
||||
bool SetOption(const std::string &key, const char* value);
|
||||
|
||||
// get option with the specified key and put to 'value',
|
||||
// return true if successful
|
||||
bool GetOption(const std::string &key, bool *value);
|
||||
bool GetOption(const std::string &key, int32 *value);
|
||||
bool GetOption(const std::string &key, uint32 *value);
|
||||
bool GetOption(const std::string &key, float *value);
|
||||
bool GetOption(const std::string &key, double *value);
|
||||
bool GetOption(const std::string &key, std::string *value);
|
||||
|
||||
enum OptionType {
|
||||
kBool,
|
||||
kInt32,
|
||||
kUint32,
|
||||
kFloat,
|
||||
kDouble,
|
||||
kString
|
||||
};
|
||||
|
||||
struct OptionInfo {
|
||||
OptionInfo(const std::string &doc, OptionType type) :
|
||||
doc(doc), type(type) {
|
||||
}
|
||||
std::string doc;
|
||||
OptionType type;
|
||||
};
|
||||
|
||||
std::vector<std::pair<std::string, OptionInfo> > GetOptionInfoList();
|
||||
|
||||
/*
|
||||
* Puts the type of the option with name 'key' in the argument 'type'.
|
||||
* Return true if such option is found, false otherwise.
|
||||
*/
|
||||
bool GetOptionType(const std::string &key, OptionType *type);
|
||||
|
||||
private:
|
||||
|
||||
std::vector<std::pair<std::string, OptionInfo> > option_info_list_;
|
||||
|
||||
// maps for option variables
|
||||
std::map<std::string, bool*> bool_map_;
|
||||
std::map<std::string, int32*> int_map_;
|
||||
std::map<std::string, uint32*> uint_map_;
|
||||
std::map<std::string, float*> float_map_;
|
||||
std::map<std::string, double*> double_map_;
|
||||
std::map<std::string, std::string*> string_map_;
|
||||
};
|
||||
|
||||
} // namespace kaldi
|
||||
|
||||
#endif // KALDI_UTIL_SIMPLE_OPTIONS_H_
|
||||
@@ -0,0 +1,275 @@
|
||||
// util/stl-utils-test.cc
|
||||
|
||||
// Copyright 2009-2012 Microsoft Corporation; Saarland University
|
||||
// Johns Hopkins University (Author: Daniel Povey)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#include "base/kaldi-common.h"
|
||||
#include "util/stl-utils.h"
|
||||
|
||||
namespace kaldi {
|
||||
static void TestIsSorted() {
|
||||
for (int i = 0;i < 100;i++) {
|
||||
std::vector<int> vec, vec2;
|
||||
int len = Rand()%5;
|
||||
for (int i = 0;i < len;i++)
|
||||
vec.push_back(Rand() % 10);
|
||||
vec2 = vec;
|
||||
std::sort(vec2.begin(), vec2.end());
|
||||
KALDI_ASSERT(IsSorted(vec) == (vec == vec2));
|
||||
}
|
||||
}
|
||||
|
||||
static void TestIsSortedAndUniq() {
|
||||
for (int i = 0;i < 100;i++) {
|
||||
std::vector<int> vec, vec2;
|
||||
int len = Rand()%5;
|
||||
for (int i = 0;i < len;i++)
|
||||
vec.push_back(Rand() % 10);
|
||||
|
||||
if (!IsSortedAndUniq(vec)) {
|
||||
bool ok = false;
|
||||
for (size_t i = 0; i+1 < (size_t)len; i++)
|
||||
if (vec[i] >= vec[i+1]) ok = true; // found out-of-order or dup.
|
||||
KALDI_ASSERT(ok);
|
||||
} else { // is sorted + uniq.
|
||||
for (size_t i = 0; i+1 < (size_t)len; i++)
|
||||
KALDI_ASSERT(vec[i] < vec[i+1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void TestUniq() {
|
||||
for (int i = 0;i < 100;i++) {
|
||||
std::vector<int> vec;
|
||||
|
||||
int cur = 1; // sorted order.
|
||||
int len = Rand()%5;
|
||||
for (int i = 0;i < len;i++) {
|
||||
cur += 1 + (Rand() % 100);
|
||||
vec.push_back(cur);
|
||||
}
|
||||
std::vector<int> vec2;
|
||||
for (int i = 0;i < len;i++) {
|
||||
int count = 1 + Rand()%5;
|
||||
for (int j = 0;j < count;j++) vec2.push_back(vec[i]);
|
||||
}
|
||||
Uniq(&vec2);
|
||||
KALDI_ASSERT(vec2 == vec);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void TestSortAndUniq() {
|
||||
for (int i = 0;i < 100;i++) {
|
||||
std::vector<int> vec;
|
||||
|
||||
int len = Rand()%5;
|
||||
for (int i = 0;i < len;i++) {
|
||||
int n = Rand() % 100;
|
||||
bool ok = true;
|
||||
for (size_t j = 0;j < vec.size();j++) if (vec[j] == n) ok = false;
|
||||
if (ok) vec.push_back(n);
|
||||
}
|
||||
// don't sort.
|
||||
std::vector<int> vec2(vec); // make sure all things in "vec" represented
|
||||
// in vec2.
|
||||
int len2 = Rand()%10;
|
||||
if (vec.size() > 0) // add more, randomly.
|
||||
for (int i = 0;i < len2;i++)
|
||||
vec2.push_back(vec[Rand()%vec.size()]);
|
||||
SortAndUniq(&vec2);
|
||||
std::sort(vec.begin(), vec.end());
|
||||
KALDI_ASSERT(vec == vec2);
|
||||
}
|
||||
}
|
||||
|
||||
void TestCopySetToVector() {
|
||||
for (int p = 0; p < 100; p++) {
|
||||
std::set<int> st;
|
||||
int sz = Rand() % 20;
|
||||
for (int i = 0;i < sz;i++) st.insert(Rand() % 10);
|
||||
std::vector<int> v;
|
||||
CopySetToVector(st, &v);
|
||||
KALDI_ASSERT(st.size() == v.size());
|
||||
for (size_t i = 0;i < v.size();i++) KALDI_ASSERT(st.count(v[i]) != 0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void TestCopyMapToVector() {
|
||||
for (int p = 0; p < 100; p++) {
|
||||
std::map<int, int> mp;
|
||||
int sz = Rand() % 20;
|
||||
for (int i = 0;i < sz;i++) mp[Rand() % 10] = Rand() % 20;
|
||||
std::vector<std::pair<int, int> > v;
|
||||
CopyMapToVector(mp, &v);
|
||||
KALDI_ASSERT(mp.size() == v.size());
|
||||
for (size_t i = 0;i < v.size();i++)
|
||||
KALDI_ASSERT(mp[v[i].first] == v[i].second);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void TestCopyMapKeysToVector() {
|
||||
for (int p = 0; p < 100; p++) {
|
||||
std::map<int, int> mp;
|
||||
int sz = Rand() % 20;
|
||||
for (int i = 0;i < sz;i++) mp[Rand() % 10] = Rand() % 20;
|
||||
std::vector<int> v;
|
||||
CopyMapKeysToVector(mp, &v);
|
||||
KALDI_ASSERT(mp.size() == v.size());
|
||||
for (size_t i = 0;i < v.size();i++) KALDI_ASSERT(mp.count(v[i]) == 1);
|
||||
}
|
||||
}
|
||||
|
||||
void TestCopyMapValuesToVector() {
|
||||
for (int p = 0; p < 100; p++) {
|
||||
std::map<int, int> mp;
|
||||
int sz = Rand() % 20;
|
||||
for (int i = 0;i < sz;i++) mp[Rand() % 10] = Rand() % 20;
|
||||
std::vector<int> v;
|
||||
CopyMapValuesToVector(mp, &v);
|
||||
KALDI_ASSERT(mp.size() == v.size());
|
||||
int i = 0;
|
||||
for (std::map<int, int>::iterator iter = mp.begin(); iter != mp.end();
|
||||
iter++) {
|
||||
KALDI_ASSERT(v[i++] == iter->second);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TestCopyMapKeysToSet() {
|
||||
for (int p = 0; p < 100; p++) {
|
||||
std::map<int, int> mp;
|
||||
int sz = Rand() % 20;
|
||||
for (int i = 0;i < sz;i++) mp[Rand() % 10] = Rand() % 20;
|
||||
std::vector<int> v;
|
||||
std::set<int> s;
|
||||
CopyMapKeysToVector(mp, &v);
|
||||
CopyMapKeysToSet(mp, &s);
|
||||
std::set<int> s2;
|
||||
CopyVectorToSet(v, &s2);
|
||||
KALDI_ASSERT(s == s2);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void TestCopyMapValuesToSet() {
|
||||
for (int p = 0; p < 100; p++) {
|
||||
std::map<int, int> mp;
|
||||
int sz = Rand() % 20;
|
||||
for (int i = 0;i < sz;i++) mp[Rand() % 10] = Rand() % 20;
|
||||
std::vector<int> v;
|
||||
std::set<int> s;
|
||||
CopyMapValuesToVector(mp, &v);
|
||||
CopyMapValuesToSet(mp, &s);
|
||||
std::set<int> s2;
|
||||
CopyVectorToSet(v, &s2);
|
||||
KALDI_ASSERT(s == s2);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void TestContainsNullPointers() {
|
||||
for (int p = 0; p < 100; p++) {
|
||||
std::vector<char*> vec;
|
||||
int sz = Rand() % 3;
|
||||
bool is_null = false;
|
||||
for (int i = 0;i < sz;i++) {
|
||||
vec.push_back(reinterpret_cast<char*>(static_cast<intptr_t>(Rand() % 2)));
|
||||
if (vec.back() == NULL)
|
||||
is_null = true;
|
||||
}
|
||||
KALDI_ASSERT(is_null == ContainsNullPointers(vec));
|
||||
}
|
||||
}
|
||||
|
||||
void TestReverseVector() {
|
||||
for (int p = 0; p < 100; p++) {
|
||||
std::vector<int> vec;
|
||||
int sz = Rand() % 5;
|
||||
for (int i = 0;i < sz;i++)
|
||||
vec.push_back(Rand() % 4);
|
||||
std::vector<int> vec2(vec), vec3(vec);
|
||||
ReverseVector(&vec2);
|
||||
ReverseVector(&vec2);
|
||||
KALDI_ASSERT(vec2 == vec);
|
||||
ReverseVector(&vec3);
|
||||
for (size_t i = 0; i < vec.size(); i++)
|
||||
KALDI_ASSERT(vec[i] == vec3[vec.size()-1-i]);
|
||||
}
|
||||
}
|
||||
|
||||
void TestMergePairVectorSumming() {
|
||||
for (int p = 0; p < 100; p++) {
|
||||
std::vector<std::pair<int32, int16> > v;
|
||||
std::map<int32, int16> m;
|
||||
int sz = Rand() % 10;
|
||||
for (size_t i = 0; i < sz; i++) {
|
||||
int32 key = Rand() % 10;
|
||||
int16 val = (Rand() % 5) - 2;
|
||||
v.push_back(std::make_pair(key, val));
|
||||
if (m.count(key) == 0) m[key] = val;
|
||||
else
|
||||
m[key] += val;
|
||||
}
|
||||
MergePairVectorSumming(&v);
|
||||
KALDI_ASSERT(IsSorted(v));
|
||||
for (size_t i = 0; i < v.size(); i++) {
|
||||
KALDI_ASSERT(v[i].second == m[v[i].first]);
|
||||
KALDI_ASSERT(v[i].second != 0.0);
|
||||
if (i > 0) KALDI_ASSERT(v[i].first > v[i-1].first);
|
||||
}
|
||||
for (std::map<int32, int16>::const_iterator iter = m.begin();
|
||||
iter != m.end(); ++iter) {
|
||||
if (iter->second != 0) {
|
||||
size_t i;
|
||||
for (i = 0; i < v.size(); i++)
|
||||
if (v[i].first == iter->first) break;
|
||||
KALDI_ASSERT(i != v.size()); // Or we didn't find this
|
||||
// key in v.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // end namespace kaldi
|
||||
|
||||
int main() {
|
||||
using namespace kaldi;
|
||||
TestIsSorted();
|
||||
TestIsSortedAndUniq();
|
||||
TestUniq();
|
||||
TestSortAndUniq();
|
||||
TestCopySetToVector();
|
||||
TestCopyMapToVector();
|
||||
TestCopyMapKeysToVector();
|
||||
TestCopyMapValuesToVector();
|
||||
TestCopyMapKeysToSet();
|
||||
TestCopyMapValuesToSet();
|
||||
TestContainsNullPointers();
|
||||
TestReverseVector();
|
||||
TestMergePairVectorSumming();
|
||||
// CopyVectorToSet implicitly tested by last 2.
|
||||
std::cout << "Test OK\n";
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
// util/stl-utils.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation; Saarland University
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_UTIL_STL_UTILS_H_
|
||||
#define KALDI_UTIL_STL_UTILS_H_
|
||||
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
using std::unordered_map;
|
||||
using std::unordered_set;
|
||||
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "base/kaldi-common.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
/// Sorts and uniq's (removes duplicates) from a vector.
|
||||
template<typename T>
|
||||
inline void SortAndUniq(std::vector<T> *vec) {
|
||||
std::sort(vec->begin(), vec->end());
|
||||
vec->erase(std::unique(vec->begin(), vec->end()), vec->end());
|
||||
}
|
||||
|
||||
|
||||
/// Returns true if the vector is sorted.
|
||||
template<typename T>
|
||||
inline bool IsSorted(const std::vector<T> &vec) {
|
||||
typename std::vector<T>::const_iterator iter = vec.begin(), end = vec.end();
|
||||
if (iter == end) return true;
|
||||
while (1) {
|
||||
typename std::vector<T>::const_iterator next_iter = iter;
|
||||
++next_iter;
|
||||
if (next_iter == end) return true; // end of loop and nothing out of order
|
||||
if (*next_iter < *iter) return false;
|
||||
iter = next_iter;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Returns true if the vector is sorted and contains each element
|
||||
/// only once.
|
||||
template<typename T>
|
||||
inline bool IsSortedAndUniq(const std::vector<T> &vec) {
|
||||
typename std::vector<T>::const_iterator iter = vec.begin(), end = vec.end();
|
||||
if (iter == end) return true;
|
||||
while (1) {
|
||||
typename std::vector<T>::const_iterator next_iter = iter;
|
||||
++next_iter;
|
||||
if (next_iter == end) return true; // end of loop and nothing out of order
|
||||
if (*next_iter <= *iter) return false;
|
||||
iter = next_iter;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Removes duplicate elements from a sorted list.
|
||||
template<typename T>
|
||||
inline void Uniq(std::vector<T> *vec) { // must be already sorted.
|
||||
KALDI_PARANOID_ASSERT(IsSorted(*vec));
|
||||
KALDI_ASSERT(vec);
|
||||
vec->erase(std::unique(vec->begin(), vec->end()), vec->end());
|
||||
}
|
||||
|
||||
/// Copies the elements of a set to a vector.
|
||||
template<class T>
|
||||
void CopySetToVector(const std::set<T> &s, std::vector<T> *v) {
|
||||
// copies members of s into v, in sorted order from lowest to highest
|
||||
// (because the set was in sorted order).
|
||||
KALDI_ASSERT(v != NULL);
|
||||
v->resize(s.size());
|
||||
typename std::set<T>::const_iterator siter = s.begin(), send = s.end();
|
||||
typename std::vector<T>::iterator viter = v->begin();
|
||||
for (; siter != send; ++siter, ++viter) {
|
||||
*viter = *siter;
|
||||
}
|
||||
}
|
||||
|
||||
template<class T>
|
||||
void CopySetToVector(const unordered_set<T> &s, std::vector<T> *v) {
|
||||
KALDI_ASSERT(v != NULL);
|
||||
v->resize(s.size());
|
||||
typename unordered_set<T>::const_iterator siter = s.begin(), send = s.end();
|
||||
typename std::vector<T>::iterator viter = v->begin();
|
||||
for (; siter != send; ++siter, ++viter) {
|
||||
*viter = *siter;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Copies the (key, value) pairs in a map to a vector of pairs.
|
||||
template<class A, class B>
|
||||
void CopyMapToVector(const std::map<A, B> &m,
|
||||
std::vector<std::pair<A, B> > *v) {
|
||||
KALDI_ASSERT(v != NULL);
|
||||
v->resize(m.size());
|
||||
typename std::map<A, B>::const_iterator miter = m.begin(), mend = m.end();
|
||||
typename std::vector<std::pair<A, B> >::iterator viter = v->begin();
|
||||
for (; miter != mend; ++miter, ++viter) {
|
||||
*viter = std::make_pair(miter->first, miter->second);
|
||||
// do it like this because of const casting.
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies the keys in a map to a vector.
|
||||
template<class A, class B>
|
||||
void CopyMapKeysToVector(const std::map<A, B> &m, std::vector<A> *v) {
|
||||
KALDI_ASSERT(v != NULL);
|
||||
v->resize(m.size());
|
||||
typename std::map<A, B>::const_iterator miter = m.begin(), mend = m.end();
|
||||
typename std::vector<A>::iterator viter = v->begin();
|
||||
for (; miter != mend; ++miter, ++viter) {
|
||||
*viter = miter->first;
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies the values in a map to a vector.
|
||||
template<class A, class B>
|
||||
void CopyMapValuesToVector(const std::map<A, B> &m, std::vector<B> *v) {
|
||||
KALDI_ASSERT(v != NULL);
|
||||
v->resize(m.size());
|
||||
typename std::map<A, B>::const_iterator miter = m.begin(), mend = m.end();
|
||||
typename std::vector<B>::iterator viter = v->begin();
|
||||
for (; miter != mend; ++miter, ++viter) {
|
||||
*viter = miter->second;
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies the keys in a map to a set.
|
||||
template<class A, class B>
|
||||
void CopyMapKeysToSet(const std::map<A, B> &m, std::set<A> *s) {
|
||||
KALDI_ASSERT(s != NULL);
|
||||
s->clear();
|
||||
typename std::map<A, B>::const_iterator miter = m.begin(), mend = m.end();
|
||||
for (; miter != mend; ++miter) {
|
||||
s->insert(s->end(), miter->first);
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies the values in a map to a set.
|
||||
template<class A, class B>
|
||||
void CopyMapValuesToSet(const std::map<A, B> &m, std::set<B> *s) {
|
||||
KALDI_ASSERT(s != NULL);
|
||||
s->clear();
|
||||
typename std::map<A, B>::const_iterator miter = m.begin(), mend = m.end();
|
||||
for (; miter != mend; ++miter)
|
||||
s->insert(s->end(), miter->second);
|
||||
}
|
||||
|
||||
|
||||
/// Copies the contents of a vector to a set.
|
||||
template<class A>
|
||||
void CopyVectorToSet(const std::vector<A> &v, std::set<A> *s) {
|
||||
KALDI_ASSERT(s != NULL);
|
||||
s->clear();
|
||||
typename std::vector<A>::const_iterator iter = v.begin(), end = v.end();
|
||||
for (; iter != end; ++iter)
|
||||
s->insert(s->end(), *iter);
|
||||
// s->end() is a hint in case v was sorted. will work regardless.
|
||||
}
|
||||
|
||||
/// Deletes any non-NULL pointers in the vector v, and sets
|
||||
/// the corresponding entries of v to NULL
|
||||
template<class A>
|
||||
void DeletePointers(std::vector<A*> *v) {
|
||||
KALDI_ASSERT(v != NULL);
|
||||
typename std::vector<A*>::iterator iter = v->begin(), end = v->end();
|
||||
for (; iter != end; ++iter) {
|
||||
if (*iter != NULL) {
|
||||
delete *iter;
|
||||
*iter = NULL; // set to NULL for extra safety.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if the vector of pointers contains NULL pointers.
|
||||
template<class A>
|
||||
bool ContainsNullPointers(const std::vector<A*> &v) {
|
||||
typename std::vector<A*>::const_iterator iter = v.begin(), end = v.end();
|
||||
for (; iter != end; ++iter)
|
||||
if (*iter == static_cast<A*> (NULL)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Copies the contents a vector of one type to a vector
|
||||
/// of another type.
|
||||
template<typename A, typename B>
|
||||
void CopyVectorToVector(const std::vector<A> &vec_in, std::vector<B> *vec_out) {
|
||||
KALDI_ASSERT(vec_out != NULL);
|
||||
vec_out->resize(vec_in.size());
|
||||
for (size_t i = 0; i < vec_in.size(); i++)
|
||||
(*vec_out)[i] = static_cast<B> (vec_in[i]);
|
||||
}
|
||||
|
||||
/// A hashing function-object for vectors.
|
||||
template<typename Int>
|
||||
struct VectorHasher { // hashing function for vector<Int>.
|
||||
size_t operator()(const std::vector<Int> &x) const noexcept {
|
||||
size_t ans = 0;
|
||||
typename std::vector<Int>::const_iterator iter = x.begin(), end = x.end();
|
||||
for (; iter != end; ++iter) {
|
||||
ans *= kPrime;
|
||||
ans += *iter;
|
||||
}
|
||||
return ans;
|
||||
}
|
||||
VectorHasher() { // Check we're instantiated with an integer type.
|
||||
KALDI_ASSERT_IS_INTEGER_TYPE(Int);
|
||||
}
|
||||
private:
|
||||
static const int kPrime = 7853;
|
||||
};
|
||||
|
||||
/// A hashing function-object for pairs of ints
|
||||
template<typename Int1, typename Int2 = Int1>
|
||||
struct PairHasher { // hashing function for pair<int>
|
||||
size_t operator()(const std::pair<Int1, Int2> &x) const noexcept {
|
||||
// 7853 was chosen at random from a list of primes.
|
||||
return x.first + x.second * 7853;
|
||||
}
|
||||
PairHasher() { // Check we're instantiated with an integer type.
|
||||
KALDI_ASSERT_IS_INTEGER_TYPE(Int1);
|
||||
KALDI_ASSERT_IS_INTEGER_TYPE(Int2);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/// A hashing function object for strings.
|
||||
struct StringHasher { // hashing function for std::string
|
||||
size_t operator()(const std::string &str) const noexcept {
|
||||
size_t ans = 0, len = str.length();
|
||||
const char *c = str.c_str(), *end = c + len;
|
||||
for (; c != end; c++) {
|
||||
ans *= kPrime;
|
||||
ans += *c;
|
||||
}
|
||||
return ans;
|
||||
}
|
||||
private:
|
||||
static const int kPrime = 7853;
|
||||
};
|
||||
|
||||
/// Reverses the contents of a vector.
|
||||
template<typename T>
|
||||
inline void ReverseVector(std::vector<T> *vec) {
|
||||
KALDI_ASSERT(vec != NULL);
|
||||
size_t sz = vec->size();
|
||||
for (size_t i = 0; i < sz/2; i++)
|
||||
std::swap( (*vec)[i], (*vec)[sz-1-i]);
|
||||
}
|
||||
|
||||
|
||||
/// Comparator object for pairs that compares only the first pair.
|
||||
template<class A, class B>
|
||||
struct CompareFirstMemberOfPair {
|
||||
inline bool operator() (const std::pair<A, B> &p1,
|
||||
const std::pair<A, B> &p2) {
|
||||
return p1.first < p2.first;
|
||||
}
|
||||
};
|
||||
|
||||
/// For a vector of pair<I, F> where I is an integer and F a floating-point or
|
||||
/// integer type, this function sorts a vector of type vector<pair<I, F> > on
|
||||
/// the I value and then merges elements with equal I values, summing these over
|
||||
/// the F component and then removing any F component with zero value. This
|
||||
/// is for where the vector of pairs represents a map from the integer to float
|
||||
/// component, with an "adding" type of semantics for combining the elements.
|
||||
template<typename I, typename F>
|
||||
inline void MergePairVectorSumming(std::vector<std::pair<I, F> > *vec) {
|
||||
KALDI_ASSERT_IS_INTEGER_TYPE(I);
|
||||
CompareFirstMemberOfPair<I, F> c;
|
||||
std::sort(vec->begin(), vec->end(), c); // sort on 1st element.
|
||||
typename std::vector<std::pair<I, F> >::iterator out = vec->begin(),
|
||||
in = vec->begin(), end = vec->end();
|
||||
// special case: while there is nothing to be changed, skip over
|
||||
// initial input (avoids unnecessary copying).
|
||||
while (in + 1 < end && in[0].first != in[1].first && in[0].second != 0.0) {
|
||||
in++;
|
||||
out++;
|
||||
}
|
||||
while (in < end) {
|
||||
// We reach this point only at the first element of
|
||||
// each stretch of identical .first elements.
|
||||
*out = *in;
|
||||
++in;
|
||||
while (in < end && in->first == out->first) {
|
||||
out->second += in->second; // this is the merge operation.
|
||||
++in;
|
||||
}
|
||||
if (out->second != static_cast<F>(0)) // Don't keep zero elements.
|
||||
out++;
|
||||
}
|
||||
vec->erase(out, end);
|
||||
}
|
||||
|
||||
} // namespace kaldi
|
||||
|
||||
#endif // KALDI_UTIL_STL_UTILS_H_
|
||||
@@ -0,0 +1,207 @@
|
||||
// util/table-types.h
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2020 Mobvoi AI Lab, Beijing, China (author: Fangjun Kuang)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#ifndef KALDI_UTIL_TABLE_TYPES_H_
|
||||
#define KALDI_UTIL_TABLE_TYPES_H_
|
||||
#include "base/kaldi-common.h"
|
||||
#include "util/kaldi-table.h"
|
||||
#include "util/kaldi-holder.h"
|
||||
#include "matrix/matrix-lib.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
// This header defines typedefs that are specific instantiations of
|
||||
// the Table types.
|
||||
|
||||
/// \addtogroup table_types
|
||||
/// @{
|
||||
|
||||
typedef TableWriter<KaldiObjectHolder<MatrixBase<BaseFloat> > >
|
||||
BaseFloatMatrixWriter;
|
||||
typedef SequentialTableReader<KaldiObjectHolder<Matrix<BaseFloat> > >
|
||||
SequentialBaseFloatMatrixReader;
|
||||
typedef RandomAccessTableReader<KaldiObjectHolder<Matrix<BaseFloat> > >
|
||||
RandomAccessBaseFloatMatrixReader;
|
||||
typedef RandomAccessTableReaderMapped<KaldiObjectHolder<Matrix<BaseFloat> > >
|
||||
RandomAccessBaseFloatMatrixReaderMapped;
|
||||
|
||||
typedef TableWriter<KaldiObjectHolder<MatrixBase<double> > >
|
||||
DoubleMatrixWriter;
|
||||
typedef SequentialTableReader<KaldiObjectHolder<Matrix<double> > >
|
||||
SequentialDoubleMatrixReader;
|
||||
typedef RandomAccessTableReader<KaldiObjectHolder<Matrix<double> > >
|
||||
RandomAccessDoubleMatrixReader;
|
||||
typedef RandomAccessTableReaderMapped<KaldiObjectHolder<Matrix<double> > >
|
||||
RandomAccessDoubleMatrixReaderMapped;
|
||||
|
||||
typedef TableWriter<KaldiObjectHolder<CompressedMatrix> >
|
||||
CompressedMatrixWriter;
|
||||
|
||||
typedef TableWriter<KaldiObjectHolder<VectorBase<BaseFloat> > >
|
||||
BaseFloatVectorWriter;
|
||||
typedef SequentialTableReader<KaldiObjectHolder<Vector<BaseFloat> > >
|
||||
SequentialBaseFloatVectorReader;
|
||||
typedef RandomAccessTableReader<KaldiObjectHolder<Vector<BaseFloat> > >
|
||||
RandomAccessBaseFloatVectorReader;
|
||||
typedef RandomAccessTableReaderMapped<KaldiObjectHolder<Vector<BaseFloat> > >
|
||||
RandomAccessBaseFloatVectorReaderMapped;
|
||||
|
||||
typedef TableWriter<KaldiObjectHolder<VectorBase<double> > >
|
||||
DoubleVectorWriter;
|
||||
typedef SequentialTableReader<KaldiObjectHolder<Vector<double> > >
|
||||
SequentialDoubleVectorReader;
|
||||
typedef RandomAccessTableReader<KaldiObjectHolder<Vector<double> > >
|
||||
RandomAccessDoubleVectorReader;
|
||||
|
||||
typedef TableWriter<KaldiObjectHolder<NumpyArray<BaseFloat>>>
|
||||
BaseFloatNumpyArrayWriter;
|
||||
typedef SequentialTableReader<KaldiObjectHolder<NumpyArray<BaseFloat>>>
|
||||
SequentialBaseFloatNumpyArrayReader;
|
||||
typedef RandomAccessTableReader<KaldiObjectHolder<NumpyArray<BaseFloat>>>
|
||||
RandomAccessBaseFloatNumpyArrayReader;
|
||||
|
||||
typedef TableWriter<KaldiObjectHolder<NumpyArray<double>>>
|
||||
DoubleNumpyArrayWriter;
|
||||
typedef SequentialTableReader<KaldiObjectHolder<NumpyArray<double>>>
|
||||
SequentialDoubleNumpyArrayReader;
|
||||
typedef RandomAccessTableReader<KaldiObjectHolder<NumpyArray<double>>>
|
||||
RandomAccessDoubleNumpyArrayReader;
|
||||
|
||||
typedef TableWriter<KaldiObjectHolder<CuMatrix<BaseFloat> > >
|
||||
BaseFloatCuMatrixWriter;
|
||||
typedef SequentialTableReader<KaldiObjectHolder<CuMatrix<BaseFloat> > >
|
||||
SequentialBaseFloatCuMatrixReader;
|
||||
typedef RandomAccessTableReader<KaldiObjectHolder<CuMatrix<BaseFloat> > >
|
||||
RandomAccessBaseFloatCuMatrixReader;
|
||||
typedef RandomAccessTableReaderMapped<KaldiObjectHolder<CuMatrix<BaseFloat> > >
|
||||
RandomAccessBaseFloatCuMatrixReaderMapped;
|
||||
|
||||
typedef TableWriter<KaldiObjectHolder<CuMatrix<double> > >
|
||||
DoubleCuMatrixWriter;
|
||||
typedef SequentialTableReader<KaldiObjectHolder<CuMatrix<double> > >
|
||||
SequentialDoubleCuMatrixReader;
|
||||
typedef RandomAccessTableReader<KaldiObjectHolder<CuMatrix<double> > >
|
||||
RandomAccessDoubleCuMatrixReader;
|
||||
typedef RandomAccessTableReaderMapped<KaldiObjectHolder<CuMatrix<double> > >
|
||||
RandomAccessDoubleCuMatrixReaderMapped;
|
||||
|
||||
typedef TableWriter<KaldiObjectHolder<CuVector<BaseFloat> > >
|
||||
BaseFloatCuVectorWriter;
|
||||
typedef SequentialTableReader<KaldiObjectHolder<CuVector<BaseFloat> > >
|
||||
SequentialBaseFloatCuVectorReader;
|
||||
typedef RandomAccessTableReader<KaldiObjectHolder<CuVector<BaseFloat> > >
|
||||
RandomAccessBaseFloatCuVectorReader;
|
||||
typedef RandomAccessTableReaderMapped<KaldiObjectHolder<CuVector<BaseFloat> > >
|
||||
RandomAccessBaseFloatCuVectorReaderMapped;
|
||||
|
||||
typedef TableWriter<KaldiObjectHolder<CuVector<double> > >
|
||||
DoubleCuVectorWriter;
|
||||
typedef SequentialTableReader<KaldiObjectHolder<CuVector<double> > >
|
||||
SequentialDoubleCuVectorReader;
|
||||
typedef RandomAccessTableReader<KaldiObjectHolder<CuVector<double> > >
|
||||
RandomAccessDoubleCuVectorReader;
|
||||
|
||||
|
||||
typedef TableWriter<BasicHolder<int32> > Int32Writer;
|
||||
typedef SequentialTableReader<BasicHolder<int32> > SequentialInt32Reader;
|
||||
typedef RandomAccessTableReader<BasicHolder<int32> > RandomAccessInt32Reader;
|
||||
|
||||
typedef TableWriter<BasicVectorHolder<int32> > Int32VectorWriter;
|
||||
typedef SequentialTableReader<BasicVectorHolder<int32> >
|
||||
SequentialInt32VectorReader;
|
||||
typedef RandomAccessTableReader<BasicVectorHolder<int32> >
|
||||
RandomAccessInt32VectorReader;
|
||||
|
||||
typedef TableWriter<BasicVectorVectorHolder<int32> > Int32VectorVectorWriter;
|
||||
typedef SequentialTableReader<BasicVectorVectorHolder<int32> >
|
||||
SequentialInt32VectorVectorReader;
|
||||
typedef RandomAccessTableReader<BasicVectorVectorHolder<int32> >
|
||||
RandomAccessInt32VectorVectorReader;
|
||||
|
||||
typedef TableWriter<BasicPairVectorHolder<int32> > Int32PairVectorWriter;
|
||||
typedef SequentialTableReader<BasicPairVectorHolder<int32> >
|
||||
SequentialInt32PairVectorReader;
|
||||
typedef RandomAccessTableReader<BasicPairVectorHolder<int32> >
|
||||
RandomAccessInt32PairVectorReader;
|
||||
|
||||
typedef TableWriter<BasicPairVectorHolder<BaseFloat> >
|
||||
BaseFloatPairVectorWriter;
|
||||
typedef SequentialTableReader<BasicPairVectorHolder<BaseFloat> >
|
||||
SequentialBaseFloatPairVectorReader;
|
||||
typedef RandomAccessTableReader<BasicPairVectorHolder<BaseFloat> >
|
||||
RandomAccessBaseFloatPairVectorReader;
|
||||
|
||||
typedef TableWriter<BasicHolder<BaseFloat> > BaseFloatWriter;
|
||||
typedef SequentialTableReader<BasicHolder<BaseFloat> >
|
||||
SequentialBaseFloatReader;
|
||||
typedef RandomAccessTableReader<BasicHolder<BaseFloat> >
|
||||
RandomAccessBaseFloatReader;
|
||||
typedef RandomAccessTableReaderMapped<BasicHolder<BaseFloat> >
|
||||
RandomAccessBaseFloatReaderMapped;
|
||||
|
||||
typedef TableWriter<BasicHolder<double> > DoubleWriter;
|
||||
typedef SequentialTableReader<BasicHolder<double> > SequentialDoubleReader;
|
||||
typedef RandomAccessTableReader<BasicHolder<double> > RandomAccessDoubleReader;
|
||||
|
||||
typedef TableWriter<BasicHolder<bool> > BoolWriter;
|
||||
typedef SequentialTableReader<BasicHolder<bool> > SequentialBoolReader;
|
||||
typedef RandomAccessTableReader<BasicHolder<bool> > RandomAccessBoolReader;
|
||||
|
||||
|
||||
|
||||
/// TokenWriter is a writer specialized for std::string where the strings
|
||||
/// are nonempty and whitespace-free. T == std::string
|
||||
typedef TableWriter<TokenHolder> TokenWriter;
|
||||
typedef SequentialTableReader<TokenHolder> SequentialTokenReader;
|
||||
typedef RandomAccessTableReader<TokenHolder> RandomAccessTokenReader;
|
||||
|
||||
|
||||
/// TokenVectorWriter is a writer specialized for sequences of
|
||||
/// std::string where the strings are nonempty and whitespace-free.
|
||||
/// T == std::vector<std::string>
|
||||
typedef TableWriter<TokenVectorHolder> TokenVectorWriter;
|
||||
// Ditto for SequentialTokenVectorReader.
|
||||
typedef SequentialTableReader<TokenVectorHolder> SequentialTokenVectorReader;
|
||||
typedef RandomAccessTableReader<TokenVectorHolder>
|
||||
RandomAccessTokenVectorReader;
|
||||
|
||||
|
||||
typedef TableWriter<KaldiObjectHolder<GeneralMatrix> >
|
||||
GeneralMatrixWriter;
|
||||
typedef SequentialTableReader<KaldiObjectHolder<GeneralMatrix> >
|
||||
SequentialGeneralMatrixReader;
|
||||
typedef RandomAccessTableReader<KaldiObjectHolder<GeneralMatrix> >
|
||||
RandomAccessGeneralMatrixReader;
|
||||
typedef RandomAccessTableReaderMapped<KaldiObjectHolder<GeneralMatrix> >
|
||||
RandomAccessGeneralMatrixReaderMapped;
|
||||
|
||||
|
||||
|
||||
/// @}
|
||||
|
||||
// Note: for FST reader/writer, see ../fstext/fstext-utils.h
|
||||
// [not done yet].
|
||||
|
||||
} // end namespace kaldi
|
||||
|
||||
|
||||
|
||||
#endif // KALDI_UTIL_TABLE_TYPES_H_
|
||||
@@ -0,0 +1,538 @@
|
||||
// util/text-utils-test.cc
|
||||
|
||||
// Copyright 2009-2011 Microsoft Corporation
|
||||
// 2017 Johns Hopkins University (author: Daniel Povey)
|
||||
// 2015 Vimal Manohar (Johns Hopkins University)
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#include "base/kaldi-common.h"
|
||||
#include "util/text-utils.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
char GetRandChar() {
|
||||
return static_cast<char>(32 + Rand() % 95); // between ' ' and '~'
|
||||
}
|
||||
|
||||
const char *ws_delim = " \t\n\r";
|
||||
char GetRandDelim() {
|
||||
if (Rand() % 2 == 0)
|
||||
return static_cast<char>(33 + Rand() % 94); // between '!' and '~';
|
||||
else
|
||||
return ws_delim[Rand() % 4];
|
||||
}
|
||||
|
||||
|
||||
void TestSplitStringToVector() {
|
||||
// srand((unsigned int)time(NULL));
|
||||
// didn't compile on cygwin.
|
||||
|
||||
{
|
||||
std::vector<std::string> str_vec;
|
||||
SplitStringToVector("", " ", false, &str_vec);
|
||||
KALDI_ASSERT(str_vec.size() == 1); // If this fails it may just mean
|
||||
// that someone changed the
|
||||
// semantics of SplitStringToVector in a reasonable way.
|
||||
SplitStringToVector("", " ", true, &str_vec);
|
||||
KALDI_ASSERT(str_vec.empty());
|
||||
}
|
||||
for (int j = 0; j < 100; j++) {
|
||||
std::vector<std::string> str_vec;
|
||||
int sz = Rand() % 73;
|
||||
std::string full;
|
||||
for (int i = 0; i < sz-1; i++) {
|
||||
full.push_back((Rand() % 7 == 0)? GetRandDelim() : GetRandChar());
|
||||
}
|
||||
std::string delim;
|
||||
delim.push_back(GetRandDelim());
|
||||
bool omit_empty_strings = (Rand() %2 == 0)? true : false;
|
||||
SplitStringToVector(full, delim.c_str(), omit_empty_strings, &str_vec);
|
||||
std::string new_full;
|
||||
for (size_t i = 0; i < str_vec.size(); i++) {
|
||||
if (omit_empty_strings) KALDI_ASSERT(str_vec[i] != "");
|
||||
new_full.append(str_vec[i]);
|
||||
if (i < str_vec.size() -1) new_full.append(delim);
|
||||
}
|
||||
std::string new_full2;
|
||||
JoinVectorToString(str_vec, delim.c_str(), omit_empty_strings, &new_full2);
|
||||
if (omit_empty_strings) { // sequences of delimiters cannot be matched
|
||||
size_t start = full.find_first_not_of(delim),
|
||||
end = full.find_last_not_of(delim);
|
||||
if (start == std::string::npos) { // only delimiters
|
||||
KALDI_ASSERT(end == std::string::npos);
|
||||
} else {
|
||||
std::string full_test;
|
||||
char last = '\0';
|
||||
for (size_t i = start; i <= end; i++) {
|
||||
if (full[i] != last || last != *delim.c_str())
|
||||
full_test.push_back(full[i]);
|
||||
last = full[i];
|
||||
}
|
||||
if (!full.empty()) {
|
||||
KALDI_ASSERT(new_full.compare(full_test) == 0);
|
||||
KALDI_ASSERT(new_full2.compare(full_test) == 0);
|
||||
}
|
||||
}
|
||||
} else if (!full.empty()) {
|
||||
KALDI_ASSERT(new_full.compare(full) == 0);
|
||||
KALDI_ASSERT(new_full2.compare(full) == 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TestSplitStringToIntegers() {
|
||||
{
|
||||
std::vector<int32> v;
|
||||
KALDI_ASSERT(SplitStringToIntegers("-1:2:4", ":", false, &v) == true
|
||||
&& v.size() == 3 && v[0] == -1 && v[1] == 2 && v[2] == 4);
|
||||
KALDI_ASSERT(SplitStringToIntegers("-1:2:4:", ":", false, &v) == false);
|
||||
KALDI_ASSERT(SplitStringToIntegers(":-1::2:4:", ":", true, &v) == true
|
||||
&& v.size() == 3 && v[0] == -1 && v[1] == 2 && v[2] == 4);
|
||||
KALDI_ASSERT(SplitStringToIntegers("-1\n2\t4", " \n\t\r", false, &v) == true
|
||||
&& v.size() == 3 && v[0] == -1 && v[1] == 2 && v[2] == 4);
|
||||
KALDI_ASSERT(SplitStringToIntegers(" ", " \n\t\r", true, &v) == true
|
||||
&& v.size() == 0);
|
||||
KALDI_ASSERT(SplitStringToIntegers("", " \n\t\r", false, &v) == true
|
||||
&& v.size() == 0);
|
||||
}
|
||||
|
||||
{
|
||||
std::vector<uint32> v;
|
||||
KALDI_ASSERT(SplitStringToIntegers("-1:2:4", ":", false, &v) == false);
|
||||
// cannot put negative number in uint32.
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
void TestSplitStringToFloats() {
|
||||
{
|
||||
std::vector<float> v;
|
||||
KALDI_ASSERT(SplitStringToFloats("-1:2.5:4", ":", false, &v) == true
|
||||
&& v.size() == 3 && v[0] == -1 && v[1] == 2.5 && v[2] == 4);
|
||||
KALDI_ASSERT(SplitStringToFloats("-1:2.5:4:", ":", false, &v) == false);
|
||||
KALDI_ASSERT(SplitStringToFloats(":-1::2:4:", ":", true, &v) == true
|
||||
&& v.size() == 3 && v[0] == -1 && v[1] == 2 && v[2] == 4);
|
||||
KALDI_ASSERT(SplitStringToFloats("-1\n2.5\t4", " \n\t\r", false, &v) == true
|
||||
&& v.size() == 3 && v[0] == -1 && v[1] == 2.5 && v[2] == 4);
|
||||
KALDI_ASSERT(SplitStringToFloats(" ", " \n\t\r", true, &v) == true
|
||||
&& v.size() == 0);
|
||||
KALDI_ASSERT(SplitStringToFloats("", " \n\t\r", false, &v) == true
|
||||
&& v.size() == 0);
|
||||
}
|
||||
|
||||
{
|
||||
std::vector<double> v;
|
||||
KALDI_ASSERT(SplitStringToFloats("-1:2:4", ":", false, &v) == true);
|
||||
}
|
||||
}
|
||||
|
||||
void TestConvertStringToInteger() {
|
||||
int32 i;
|
||||
KALDI_ASSERT(ConvertStringToInteger("12345", &i) && i == 12345);
|
||||
KALDI_ASSERT(ConvertStringToInteger("-12345", &i) && i == -12345);
|
||||
char j;
|
||||
KALDI_ASSERT(!ConvertStringToInteger("-12345", &j)); // too big for char.
|
||||
|
||||
KALDI_ASSERT(ConvertStringToInteger(" -12345 ", &i)); // whitespace accepted
|
||||
|
||||
KALDI_ASSERT(!ConvertStringToInteger("a ", &i)); // non-integers rejected.
|
||||
|
||||
KALDI_ASSERT(ConvertStringToInteger("0", &i) && i == 0);
|
||||
|
||||
uint64 k;
|
||||
KALDI_ASSERT(ConvertStringToInteger("12345", &k) && k == 12345);
|
||||
KALDI_ASSERT(!ConvertStringToInteger("-12345", &k)); // unsigned,
|
||||
// cannot convert.
|
||||
}
|
||||
|
||||
template<class Real>
|
||||
void TestConvertStringToReal() {
|
||||
Real d;
|
||||
KALDI_ASSERT(ConvertStringToReal("1", &d) && d == 1.0);
|
||||
KALDI_ASSERT(ConvertStringToReal("-1", &d) && d == -1.0);
|
||||
KALDI_ASSERT(ConvertStringToReal("-1", &d) && d == -1.0);
|
||||
KALDI_ASSERT(ConvertStringToReal(" -1 ", &d) && d == -1.0);
|
||||
KALDI_ASSERT(!ConvertStringToReal("-1 x", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("-1f", &d));
|
||||
KALDI_ASSERT(ConvertStringToReal("12345.2", &d) && fabs(d-12345.2) < 1.0);
|
||||
KALDI_ASSERT(ConvertStringToReal("1.0e+08", &d) && fabs(d-1.0e+08) < 100.0);
|
||||
|
||||
// it also works for inf or nan.
|
||||
KALDI_ASSERT(ConvertStringToReal("inf", &d) && d > 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal(" inf", &d) && d > 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal("inf ", &d) && d > 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal(" inf ", &d) && d > 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal("+inf", &d) && d > 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal("-inf", &d) && d < 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal("Inf", &d) && d > 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal("INF", &d) && d > 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal("InF", &d) && d > 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal("infinity", &d) && d > 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal("-infinity", &d) && d < 0 && d - d != 0);
|
||||
KALDI_ASSERT(!ConvertStringToReal("GARBAGE inf", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("GARBAGEinf", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("infGARBAGE", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("inf_GARBAGE", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("inf GARBAGE", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("GARBAGE infinity", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("GARBAGEinfinity", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("infinityGARBAGE", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("infinity_GARBAGE", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("infinity GARBAGE", &d));
|
||||
KALDI_ASSERT(ConvertStringToReal("1.#INF", &d) && d > 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal("-1.#INF", &d) && d < 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal("-1.#INF ", &d) && d < 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal(" -1.#INF ", &d) && d < 0 && d - d != 0);
|
||||
KALDI_ASSERT(!ConvertStringToReal("GARBAGE 1.#INF", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("GARBAGE1.#INF", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("2.#INF", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("-2.#INF", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("1.#INFGARBAGE", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("1.#INF_GARBAGE", &d));
|
||||
|
||||
KALDI_ASSERT(ConvertStringToReal("nan", &d) && d != d);
|
||||
KALDI_ASSERT(ConvertStringToReal("+nan", &d) && d != d);
|
||||
KALDI_ASSERT(ConvertStringToReal("-nan", &d) && d != d);
|
||||
KALDI_ASSERT(ConvertStringToReal("Nan", &d) && d != d);
|
||||
KALDI_ASSERT(ConvertStringToReal("NAN", &d) && d != d);
|
||||
KALDI_ASSERT(ConvertStringToReal("NaN", &d) && d != d);
|
||||
KALDI_ASSERT(ConvertStringToReal(" NaN", &d) && d != d);
|
||||
KALDI_ASSERT(ConvertStringToReal("NaN ", &d) && d != d);
|
||||
KALDI_ASSERT(ConvertStringToReal(" NaN ", &d) && d != d);
|
||||
KALDI_ASSERT(ConvertStringToReal("1.#QNAN", &d) && d != d);
|
||||
KALDI_ASSERT(ConvertStringToReal("-1.#QNAN", &d) && d != d);
|
||||
KALDI_ASSERT(ConvertStringToReal("1.#QNAN ", &d) && d != d);
|
||||
KALDI_ASSERT(ConvertStringToReal(" 1.#QNAN ", &d) && d != d);
|
||||
KALDI_ASSERT(!ConvertStringToReal("GARBAGE nan", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("GARBAGEnan", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("nanGARBAGE", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("nan_GARBAGE", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("nan GARBAGE", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("GARBAGE 1.#QNAN", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("GARBAGE1.#QNAN", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("2.#QNAN", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("-2.#QNAN", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("-1.#QNAN_GARBAGE", &d));
|
||||
KALDI_ASSERT(!ConvertStringToReal("-1.#QNANGARBAGE", &d));
|
||||
}
|
||||
|
||||
template<class Real>
|
||||
void TestNan() {
|
||||
Real d;
|
||||
KALDI_ASSERT(ConvertStringToReal(std::to_string(sqrt(-1)), &d) && d != d);
|
||||
}
|
||||
|
||||
template<class Real>
|
||||
void TestInf() {
|
||||
Real d;
|
||||
KALDI_ASSERT(ConvertStringToReal(std::to_string(exp(10000)), &d) &&
|
||||
d > 0 && d - d != 0);
|
||||
KALDI_ASSERT(ConvertStringToReal(std::to_string(-exp(10000)), &d) &&
|
||||
d < 0 && d - d != 0);
|
||||
}
|
||||
|
||||
|
||||
std::string TrimTmp(std::string s) {
|
||||
Trim(&s);
|
||||
return s;
|
||||
}
|
||||
|
||||
void TestTrim() {
|
||||
KALDI_ASSERT(TrimTmp(" a ") == "a");
|
||||
KALDI_ASSERT(TrimTmp(" a b c") == "a b c");
|
||||
KALDI_ASSERT(TrimTmp("") == "");
|
||||
KALDI_ASSERT(TrimTmp("X\n") == "X");
|
||||
KALDI_ASSERT(TrimTmp("X\n\t") == "X");
|
||||
KALDI_ASSERT(TrimTmp("\n\tX") == "X");
|
||||
} // end namespace kaldi
|
||||
|
||||
|
||||
void TestSplitStringOnFirstSpace() {
|
||||
std::string a, b;
|
||||
SplitStringOnFirstSpace("a b", &a, &b);
|
||||
KALDI_ASSERT(a == "a" && b == "b");
|
||||
SplitStringOnFirstSpace("aa bb", &a, &b);
|
||||
KALDI_ASSERT(a == "aa" && b == "bb");
|
||||
SplitStringOnFirstSpace("aa", &a, &b);
|
||||
KALDI_ASSERT(a == "aa" && b == "");
|
||||
SplitStringOnFirstSpace(" aa \n\t ", &a, &b);
|
||||
KALDI_ASSERT(a == "aa" && b == "");
|
||||
SplitStringOnFirstSpace(" \n\t ", &a, &b);
|
||||
KALDI_ASSERT(a == "" && b == "");
|
||||
SplitStringOnFirstSpace(" aa bb \n\t ", &a, &b);
|
||||
KALDI_ASSERT(a == "aa" && b == "bb");
|
||||
SplitStringOnFirstSpace(" aa bb cc ", &a, &b);
|
||||
KALDI_ASSERT(a == "aa" && b == "bb cc");
|
||||
SplitStringOnFirstSpace(" aa bb cc ", &a, &b);
|
||||
KALDI_ASSERT(a == "aa" && b == "bb cc");
|
||||
SplitStringOnFirstSpace(" aa bb cc", &a, &b);
|
||||
KALDI_ASSERT(a == "aa" && b == "bb cc");
|
||||
}
|
||||
|
||||
void TestIsToken() {
|
||||
KALDI_ASSERT(IsToken("a"));
|
||||
KALDI_ASSERT(IsToken("ab"));
|
||||
KALDI_ASSERT(!IsToken("ab "));
|
||||
KALDI_ASSERT(!IsToken(" ab"));
|
||||
KALDI_ASSERT(!IsToken("a b"));
|
||||
KALDI_ASSERT(IsToken("\231")); // typical non-ASCII printable character,
|
||||
// something with an accent.
|
||||
KALDI_ASSERT(!IsToken("\377")); // character 255, which is a form of space.
|
||||
KALDI_ASSERT(IsToken("a-b,c,d=ef"));
|
||||
KALDI_ASSERT(!IsToken("a\nb"));
|
||||
KALDI_ASSERT(!IsToken("a\tb"));
|
||||
KALDI_ASSERT(!IsToken("ab\t"));
|
||||
KALDI_ASSERT(!IsToken(""));
|
||||
}
|
||||
|
||||
void TestIsLine() {
|
||||
KALDI_ASSERT(IsLine("a"));
|
||||
KALDI_ASSERT(IsLine("a b"));
|
||||
KALDI_ASSERT(!IsLine("a\nb"));
|
||||
KALDI_ASSERT(!IsLine("a b "));
|
||||
KALDI_ASSERT(!IsLine(" a b"));
|
||||
}
|
||||
|
||||
|
||||
void TestStringsApproxEqual() {
|
||||
// we must test the test.
|
||||
KALDI_ASSERT(!StringsApproxEqual("a", "b"));
|
||||
KALDI_ASSERT(!StringsApproxEqual("1", "2"));
|
||||
KALDI_ASSERT(StringsApproxEqual("1.234", "1.235", 2));
|
||||
KALDI_ASSERT(!StringsApproxEqual("1.234", "1.235", 3));
|
||||
KALDI_ASSERT(StringsApproxEqual("x 1.234 y", "x 1.2345 y", 3));
|
||||
KALDI_ASSERT(!StringsApproxEqual("x 1.234 y", "x 1.2345 y", 4));
|
||||
KALDI_ASSERT(StringsApproxEqual("x 1.234 y 6.41", "x 1.235 y 6.49", 1));
|
||||
KALDI_ASSERT(!StringsApproxEqual("x 1.234 y 6.41", "x 1.235 y 6.49", 2));
|
||||
KALDI_ASSERT(StringsApproxEqual("x 1.234 y 6.41", "x 1.235 y 6.411", 2));
|
||||
KALDI_ASSERT(StringsApproxEqual("x 1.0 y", "x 1.0001 y", 3));
|
||||
KALDI_ASSERT(!StringsApproxEqual("x 1.0 y", "x 1.0001 y", 4));
|
||||
}
|
||||
|
||||
void UnitTestConfigLineParse() {
|
||||
std::string str;
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "a-b xx=yyy foo=bar baz=123 ba=1:2";
|
||||
bool status = cfl.ParseLine(str);
|
||||
KALDI_ASSERT(status && cfl.FirstToken() == "a-b");
|
||||
|
||||
KALDI_ASSERT(cfl.HasUnusedValues());
|
||||
std::string str_value;
|
||||
KALDI_ASSERT(cfl.GetValue("xx", &str_value));
|
||||
KALDI_ASSERT(str_value == "yyy");
|
||||
KALDI_ASSERT(cfl.HasUnusedValues());
|
||||
KALDI_ASSERT(cfl.GetValue("foo", &str_value));
|
||||
KALDI_ASSERT(str_value == "bar");
|
||||
KALDI_ASSERT(cfl.HasUnusedValues());
|
||||
KALDI_ASSERT(!cfl.GetValue("xy", &str_value));
|
||||
KALDI_ASSERT(cfl.GetValue("baz", &str_value));
|
||||
KALDI_ASSERT(str_value == "123");
|
||||
|
||||
std::vector<int32> int_values;
|
||||
KALDI_ASSERT(!cfl.GetValue("xx", &int_values));
|
||||
KALDI_ASSERT(cfl.GetValue("baz", &int_values));
|
||||
KALDI_ASSERT(cfl.HasUnusedValues());
|
||||
KALDI_ASSERT(int_values.size() == 1 && int_values[0] == 123);
|
||||
KALDI_ASSERT(cfl.GetValue("ba", &int_values));
|
||||
KALDI_ASSERT(int_values.size() == 2 && int_values[0] == 1 && int_values[1] == 2);
|
||||
KALDI_ASSERT(!cfl.HasUnusedValues());
|
||||
}
|
||||
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "a-b baz=x y z pp = qq ab =cd ac= bd";
|
||||
KALDI_ASSERT(!cfl.ParseLine(str));
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "a-b baz=x y z pp = qq ab=cd ac=bd";
|
||||
KALDI_ASSERT(!cfl.ParseLine(str));
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "foo-bar";
|
||||
KALDI_ASSERT(cfl.ParseLine(str));
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "foo-bar a=b c d f=g";
|
||||
std::string value;
|
||||
KALDI_ASSERT(cfl.ParseLine(str) && cfl.FirstToken() == "foo-bar" &&
|
||||
cfl.GetValue("a", &value) && value == "b c d" &&
|
||||
cfl.GetValue("f", &value) && value == "g" &&
|
||||
!cfl.HasUnusedValues());
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "zzz a=b baz";
|
||||
KALDI_ASSERT(cfl.ParseLine(str) && cfl.FirstToken() == "zzz" &&
|
||||
cfl.UnusedValues() == "a=b baz");
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "xxx a=b baz ";
|
||||
KALDI_ASSERT(cfl.ParseLine(str) && cfl.UnusedValues() == "a=b baz");
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "xxx a=b =c";
|
||||
KALDI_ASSERT(!cfl.ParseLine(str));
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "xxx baz='x y z' pp=qq ab=cd ac=bd";
|
||||
KALDI_ASSERT(cfl.ParseLine(str) && cfl.FirstToken() == "xxx");
|
||||
std::string str_value;
|
||||
KALDI_ASSERT(cfl.GetValue("baz", &str_value));
|
||||
KALDI_ASSERT(str_value == "x y z");
|
||||
KALDI_ASSERT(cfl.GetValue("pp", &str_value));
|
||||
KALDI_ASSERT(str_value == "qq");
|
||||
KALDI_ASSERT(cfl.UnusedValues() == "ab=cd ac=bd");
|
||||
KALDI_ASSERT(cfl.GetValue("ab", &str_value));
|
||||
KALDI_ASSERT(str_value == "cd");
|
||||
KALDI_ASSERT(cfl.UnusedValues() == "ac=bd");
|
||||
KALDI_ASSERT(cfl.HasUnusedValues());
|
||||
KALDI_ASSERT(cfl.GetValue("ac", &str_value));
|
||||
KALDI_ASSERT(str_value == "bd");
|
||||
KALDI_ASSERT(!cfl.HasUnusedValues());
|
||||
}
|
||||
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "x baz= pp = qq flag=t ";
|
||||
KALDI_ASSERT(!cfl.ParseLine(str));
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = " x baz= pp=qq flag=t ";
|
||||
KALDI_ASSERT(cfl.ParseLine(str) && cfl.FirstToken() == "x");
|
||||
|
||||
std::string str_value;
|
||||
KALDI_ASSERT(cfl.GetValue("baz", &str_value));
|
||||
KALDI_ASSERT(str_value == "");
|
||||
KALDI_ASSERT(cfl.GetValue("pp", &str_value));
|
||||
KALDI_ASSERT(str_value == "qq");
|
||||
KALDI_ASSERT(cfl.HasUnusedValues());
|
||||
KALDI_ASSERT(cfl.GetValue("flag", &str_value));
|
||||
KALDI_ASSERT(str_value == "t");
|
||||
KALDI_ASSERT(!cfl.HasUnusedValues());
|
||||
|
||||
bool bool_value = false;
|
||||
KALDI_ASSERT(cfl.GetValue("flag", &bool_value));
|
||||
KALDI_ASSERT(bool_value);
|
||||
}
|
||||
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "xx _baz=a -pp=qq";
|
||||
KALDI_ASSERT(!cfl.ParseLine(str));
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "xx 0baz=a pp=qq";
|
||||
KALDI_ASSERT(!cfl.ParseLine(str));
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "xx -baz=a pp=qq";
|
||||
KALDI_ASSERT(!cfl.ParseLine(str));
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "xx _baz'=a pp=qq";
|
||||
KALDI_ASSERT(!cfl.ParseLine(str));
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = " baz=g";
|
||||
KALDI_ASSERT(cfl.ParseLine(str) && cfl.FirstToken() == "");
|
||||
bool flag;
|
||||
KALDI_ASSERT(!cfl.GetValue("baz", &flag));
|
||||
}
|
||||
{
|
||||
ConfigLine cfl;
|
||||
str = "xx _baz1=a pp=qq";
|
||||
KALDI_ASSERT(cfl.ParseLine(str));
|
||||
|
||||
std::string str_value;
|
||||
KALDI_ASSERT(cfl.GetValue("_baz1", &str_value));
|
||||
}
|
||||
}
|
||||
|
||||
void UnitTestReadConfig() {
|
||||
std::string str = "a-b alpha=aa beta=\"b b\"# String test\n"
|
||||
"a-b beta2='b c' beta3=bd # \n"
|
||||
"a-b gamma=1:2:3:4 # Int Vector test\n"
|
||||
" a-b de1ta=f # Bool + Integer in key Comment test delta=t \n"
|
||||
"a-b _epsilon=-1 # Int Vector test _epsilon=1 \n"
|
||||
"a-b zet-_a=0.15 theta=1.1# Float, -, _ test\n"
|
||||
"a-b quoted='a b c' # quoted string\n"
|
||||
"a-b quoted2=\"d e 'a b=c' f\" # string quoted with double quotes";
|
||||
|
||||
std::istringstream is(str);
|
||||
std::vector<std::string> lines;
|
||||
ReadConfigLines(is, &lines);
|
||||
KALDI_ASSERT(lines.size() == 8);
|
||||
|
||||
ConfigLine cfl;
|
||||
for (size_t i = 0; i < lines.size(); i++) {
|
||||
KALDI_ASSERT(cfl.ParseLine(lines[i]) && cfl.FirstToken() == "a-b");
|
||||
if (i == 1) {
|
||||
KALDI_ASSERT(cfl.GetValue("beta2", &str) && str == "b c");
|
||||
}
|
||||
if (i == 4) {
|
||||
KALDI_ASSERT(cfl.GetValue("_epsilon", &str) && str == "-1");
|
||||
}
|
||||
if (i == 5) {
|
||||
BaseFloat float_val = 0;
|
||||
KALDI_ASSERT(cfl.GetValue("zet-_a", &float_val) && ApproxEqual(float_val, 0.15));
|
||||
}
|
||||
if (i == 6) {
|
||||
KALDI_ASSERT(cfl.GetValue("quoted", &str) && str == "a b c");
|
||||
}
|
||||
if (i == 7) {
|
||||
KALDI_ASSERT(cfl.GetValue("quoted2", &str) && str == "d e 'a b=c' f");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // end namespace kaldi
|
||||
|
||||
int main() {
|
||||
using namespace kaldi;
|
||||
TestSplitStringToVector();
|
||||
TestSplitStringToIntegers();
|
||||
TestSplitStringToFloats();
|
||||
TestConvertStringToInteger();
|
||||
TestConvertStringToReal<float>();
|
||||
TestConvertStringToReal<double>();
|
||||
TestTrim();
|
||||
TestSplitStringOnFirstSpace();
|
||||
TestIsToken();
|
||||
TestIsLine();
|
||||
TestStringsApproxEqual();
|
||||
TestNan<float>();
|
||||
TestNan<double>();
|
||||
TestInf<float>();
|
||||
TestInf<double>();
|
||||
UnitTestConfigLineParse();
|
||||
UnitTestReadConfig();
|
||||
std::cout << "Test OK\n";
|
||||
}
|
||||
@@ -0,0 +1,591 @@
|
||||
// util/text-utils.cc
|
||||
|
||||
// Copyright 2009-2011 Saarland University; Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "util/text-utils.h"
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <algorithm>
|
||||
#include "base/kaldi-common.h"
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
|
||||
template<class F>
|
||||
bool SplitStringToFloats(const std::string &full,
|
||||
const char *delim,
|
||||
bool omit_empty_strings, // typically false
|
||||
std::vector<F> *out) {
|
||||
KALDI_ASSERT(out != NULL);
|
||||
if (*(full.c_str()) == '\0') {
|
||||
out->clear();
|
||||
return true;
|
||||
}
|
||||
std::vector<std::string> split;
|
||||
SplitStringToVector(full, delim, omit_empty_strings, &split);
|
||||
out->resize(split.size());
|
||||
for (size_t i = 0; i < split.size(); i++) {
|
||||
F f = 0;
|
||||
if (!ConvertStringToReal(split[i], &f))
|
||||
return false;
|
||||
(*out)[i] = f;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Instantiate the template above for float and double.
|
||||
template
|
||||
bool SplitStringToFloats(const std::string &full,
|
||||
const char *delim,
|
||||
bool omit_empty_strings,
|
||||
std::vector<float> *out);
|
||||
template
|
||||
bool SplitStringToFloats(const std::string &full,
|
||||
const char *delim,
|
||||
bool omit_empty_strings,
|
||||
std::vector<double> *out);
|
||||
|
||||
void SplitStringToVector(const std::string &full, const char *delim,
|
||||
bool omit_empty_strings,
|
||||
std::vector<std::string> *out) {
|
||||
size_t start = 0, found = 0, end = full.size();
|
||||
out->clear();
|
||||
while (found != std::string::npos) {
|
||||
found = full.find_first_of(delim, start);
|
||||
// start != end condition is for when the delimiter is at the end
|
||||
if (!omit_empty_strings || (found != start && start != end))
|
||||
out->push_back(full.substr(start, found - start));
|
||||
start = found + 1;
|
||||
}
|
||||
}
|
||||
|
||||
void JoinVectorToString(const std::vector<std::string> &vec_in,
|
||||
const char *delim, bool omit_empty_strings,
|
||||
std::string *str_out) {
|
||||
std::string tmp_str;
|
||||
for (size_t i = 0; i < vec_in.size(); i++) {
|
||||
if (!omit_empty_strings || !vec_in[i].empty()) {
|
||||
tmp_str.append(vec_in[i]);
|
||||
if (i < vec_in.size() - 1)
|
||||
if (!omit_empty_strings || !vec_in[i+1].empty())
|
||||
tmp_str.append(delim);
|
||||
}
|
||||
}
|
||||
str_out->swap(tmp_str);
|
||||
}
|
||||
|
||||
void Trim(std::string *str) {
|
||||
const char *white_chars = " \t\n\r\f\v";
|
||||
|
||||
std::string::size_type pos = str->find_last_not_of(white_chars);
|
||||
if (pos != std::string::npos) {
|
||||
str->erase(pos + 1);
|
||||
pos = str->find_first_not_of(white_chars);
|
||||
if (pos != std::string::npos) str->erase(0, pos);
|
||||
} else {
|
||||
str->erase(str->begin(), str->end());
|
||||
}
|
||||
}
|
||||
|
||||
bool IsToken(const std::string &token) {
|
||||
size_t l = token.length();
|
||||
if (l == 0) return false;
|
||||
for (size_t i = 0; i < l; i++) {
|
||||
unsigned char c = token[i];
|
||||
if ((!isprint(c) || isspace(c)) && (isascii(c) || c == (unsigned char)255))
|
||||
return false;
|
||||
// The "&& (isascii(c) || c == 255)" was added so that we won't reject
|
||||
// non-ASCII characters such as French characters with accents [except for
|
||||
// 255 which is "nbsp", a form of space].
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void SplitStringOnFirstSpace(const std::string &str,
|
||||
std::string *first,
|
||||
std::string *rest) {
|
||||
const char *white_chars = " \t\n\r\f\v";
|
||||
typedef std::string::size_type I;
|
||||
const I npos = std::string::npos;
|
||||
I first_nonwhite = str.find_first_not_of(white_chars);
|
||||
if (first_nonwhite == npos) {
|
||||
first->clear();
|
||||
rest->clear();
|
||||
return;
|
||||
}
|
||||
// next_white is first whitespace after first nonwhitespace.
|
||||
I next_white = str.find_first_of(white_chars, first_nonwhite);
|
||||
|
||||
if (next_white == npos) { // no more whitespace...
|
||||
*first = std::string(str, first_nonwhite);
|
||||
rest->clear();
|
||||
return;
|
||||
}
|
||||
I next_nonwhite = str.find_first_not_of(white_chars, next_white);
|
||||
if (next_nonwhite == npos) {
|
||||
*first = std::string(str, first_nonwhite, next_white-first_nonwhite);
|
||||
rest->clear();
|
||||
return;
|
||||
}
|
||||
|
||||
I last_nonwhite = str.find_last_not_of(white_chars);
|
||||
KALDI_ASSERT(last_nonwhite != npos); // or coding error.
|
||||
|
||||
*first = std::string(str, first_nonwhite, next_white-first_nonwhite);
|
||||
*rest = std::string(str, next_nonwhite, last_nonwhite+1-next_nonwhite);
|
||||
}
|
||||
|
||||
bool IsLine(const std::string &line) {
|
||||
if (line.find('\n') != std::string::npos) return false;
|
||||
if (line.empty()) return true;
|
||||
if (isspace(*(line.begin()))) return false;
|
||||
if (isspace(*(line.rbegin()))) return false;
|
||||
std::string::const_iterator iter = line.begin(), end = line.end();
|
||||
for (; iter != end; iter++)
|
||||
if (!isprint(*iter)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
class NumberIstream{
|
||||
public:
|
||||
explicit NumberIstream(std::istream &i) : in_(i) {}
|
||||
|
||||
NumberIstream & operator >> (T &x) {
|
||||
if (!in_.good()) return *this;
|
||||
in_ >> x;
|
||||
if (!in_.fail() && RemainderIsOnlySpaces()) return *this;
|
||||
return ParseOnFail(&x);
|
||||
}
|
||||
|
||||
private:
|
||||
std::istream &in_;
|
||||
|
||||
bool RemainderIsOnlySpaces() {
|
||||
if (in_.tellg() != std::istream::pos_type(-1)) {
|
||||
std::string rem;
|
||||
in_ >> rem;
|
||||
|
||||
if (rem.find_first_not_of(' ') != std::string::npos) {
|
||||
// there is not only spaces
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
in_.clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
NumberIstream & ParseOnFail(T *x) {
|
||||
std::string str;
|
||||
in_.clear();
|
||||
in_.seekg(0);
|
||||
// If the stream is broken even before trying
|
||||
// to read from it or if there are many tokens,
|
||||
// it's pointless to try.
|
||||
if (!(in_ >> str) || !RemainderIsOnlySpaces()) {
|
||||
in_.setstate(std::ios_base::failbit);
|
||||
return *this;
|
||||
}
|
||||
|
||||
std::map<std::string, T> inf_nan_map;
|
||||
// we'll keep just uppercase values.
|
||||
inf_nan_map["INF"] = std::numeric_limits<T>::infinity();
|
||||
inf_nan_map["+INF"] = std::numeric_limits<T>::infinity();
|
||||
inf_nan_map["-INF"] = - std::numeric_limits<T>::infinity();
|
||||
inf_nan_map["INFINITY"] = std::numeric_limits<T>::infinity();
|
||||
inf_nan_map["+INFINITY"] = std::numeric_limits<T>::infinity();
|
||||
inf_nan_map["-INFINITY"] = - std::numeric_limits<T>::infinity();
|
||||
inf_nan_map["NAN"] = std::numeric_limits<T>::quiet_NaN();
|
||||
inf_nan_map["+NAN"] = std::numeric_limits<T>::quiet_NaN();
|
||||
inf_nan_map["-NAN"] = - std::numeric_limits<T>::quiet_NaN();
|
||||
// MSVC
|
||||
inf_nan_map["1.#INF"] = std::numeric_limits<T>::infinity();
|
||||
inf_nan_map["-1.#INF"] = - std::numeric_limits<T>::infinity();
|
||||
inf_nan_map["1.#QNAN"] = std::numeric_limits<T>::quiet_NaN();
|
||||
inf_nan_map["-1.#QNAN"] = - std::numeric_limits<T>::quiet_NaN();
|
||||
|
||||
std::transform(str.begin(), str.end(), str.begin(), ::toupper);
|
||||
|
||||
if (inf_nan_map.find(str) != inf_nan_map.end()) {
|
||||
*x = inf_nan_map[str];
|
||||
} else {
|
||||
in_.setstate(std::ios_base::failbit);
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <typename T>
|
||||
bool ConvertStringToReal(const std::string &str,
|
||||
T *out) {
|
||||
std::istringstream iss(str);
|
||||
|
||||
NumberIstream<T> i(iss);
|
||||
|
||||
i >> *out;
|
||||
|
||||
if (iss.fail()) {
|
||||
// Number conversion failed.
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
template
|
||||
bool ConvertStringToReal(const std::string &str,
|
||||
float *out);
|
||||
template
|
||||
bool ConvertStringToReal(const std::string &str,
|
||||
double *out);
|
||||
|
||||
|
||||
|
||||
/*
|
||||
This function is a helper function of StringsApproxEqual. It should be
|
||||
thought of as a recursive function-- it was designed that way-- but rather
|
||||
than actually recursing (which would cause problems with stack overflow), we
|
||||
just set the args and return to the start.
|
||||
|
||||
The 'decimal_places_tolerance' argument is just passed in from outside,
|
||||
see the documentation for StringsApproxEqual in text-utils.h to see an
|
||||
explanation. The argument 'places_into_number' provides some information
|
||||
about the strings 'a' and 'b' that precedes the current pointers.
|
||||
For purposes of this comment, let's define the 'decimal' of a number
|
||||
as the part that comes after the decimal point, e.g. in '99.123',
|
||||
'123' would be the decimal. If 'places_into_number' is -1, it means
|
||||
we're not currently inside some place like that (i.e. it's not the
|
||||
case that we're pointing to the '1' or the '2' or the '3').
|
||||
If it's 0, then we'd be pointing to the first place after the decimal,
|
||||
'1' in this case. Note if one of the numbers is shorter than the
|
||||
other, like '99.123' versus '99.1234' and 'a' points to the first '3'
|
||||
while 'b' points to the second '4', 'places_into_number' referes to the
|
||||
shorter of the two, i.e. it would be 2 in this example.
|
||||
|
||||
|
||||
*/
|
||||
bool StringsApproxEqualInternal(const char *a, const char *b,
|
||||
int32 decimal_places_tolerance,
|
||||
int32 places_into_number) {
|
||||
start:
|
||||
char ca = *a, cb = *b;
|
||||
if (ca == cb) {
|
||||
if (ca == '\0') {
|
||||
return true;
|
||||
} else {
|
||||
if (places_into_number >= 0) {
|
||||
if (isdigit(ca)) {
|
||||
places_into_number++;
|
||||
} else {
|
||||
places_into_number = -1;
|
||||
}
|
||||
} else {
|
||||
if (ca == '.') {
|
||||
places_into_number = 0;
|
||||
}
|
||||
}
|
||||
a++;
|
||||
b++;
|
||||
goto start;
|
||||
}
|
||||
} else {
|
||||
if (places_into_number >= decimal_places_tolerance &&
|
||||
(isdigit(ca) || isdigit(cb))) {
|
||||
// we're potentially willing to accept this difference between the
|
||||
// strings.
|
||||
if (isdigit(ca)) a++;
|
||||
if (isdigit(cb)) b++;
|
||||
// we'll have advanced at least one of the two strings.
|
||||
goto start;
|
||||
} else if (places_into_number >= 0 &&
|
||||
((ca == '0' && !isdigit(cb)) || (cb == '0' && !isdigit(ca)))) {
|
||||
// this clause is designed to ensure that, for example,
|
||||
// "0.1" would count the same as "0.100001".
|
||||
if (ca == '0') a++;
|
||||
else b++;
|
||||
places_into_number++;
|
||||
goto start;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
bool StringsApproxEqual(const std::string &a,
|
||||
const std::string &b,
|
||||
int32 decimal_places_tolerance) {
|
||||
return StringsApproxEqualInternal(a.c_str(), b.c_str(),
|
||||
decimal_places_tolerance, -1);
|
||||
}
|
||||
|
||||
|
||||
bool ConfigLine::ParseLine(const std::string &line) {
|
||||
data_.clear();
|
||||
whole_line_ = line;
|
||||
if (line.size() == 0) return false; // Empty line
|
||||
size_t pos = 0, size = line.size();
|
||||
while (isspace(line[pos]) && pos < size) pos++;
|
||||
if (pos == size)
|
||||
return false; // whitespace-only line
|
||||
size_t first_token_start_pos = pos;
|
||||
// first get first_token_.
|
||||
while (!isspace(line[pos]) && pos < size) {
|
||||
if (line[pos] == '=') {
|
||||
// If the first block of non-whitespace looks like "foo-bar=...",
|
||||
// then we ignore it: there is no initial token, and FirstToken()
|
||||
// is empty.
|
||||
pos = first_token_start_pos;
|
||||
break;
|
||||
}
|
||||
pos++;
|
||||
}
|
||||
first_token_ = std::string(line, first_token_start_pos, pos - first_token_start_pos);
|
||||
// first_token_ is expected to be either empty or something like
|
||||
// "component-node", which actually is a slightly more restrictive set of
|
||||
// strings than IsValidName() checks for this is a convenient way to check it.
|
||||
if (!first_token_.empty() && !IsValidName(first_token_))
|
||||
return false;
|
||||
|
||||
while (pos < size) {
|
||||
if (isspace(line[pos])) {
|
||||
pos++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// OK, at this point we know that we are pointing at nonspace.
|
||||
size_t next_equals_sign = line.find_first_of("=", pos);
|
||||
if (next_equals_sign == pos || next_equals_sign == std::string::npos) {
|
||||
// we're looking for something like 'key=value'. If there is no equals sign,
|
||||
// or it's not preceded by something, it's a parsing failure.
|
||||
return false;
|
||||
}
|
||||
std::string key(line, pos, next_equals_sign - pos);
|
||||
if (!IsValidName(key)) return false;
|
||||
|
||||
// handle any quotes. we support key='blah blah' or key="foo bar".
|
||||
// no escaping is supported.
|
||||
if (line[next_equals_sign+1] == '\'' || line[next_equals_sign+1] == '"') {
|
||||
char my_quote = line[next_equals_sign+1];
|
||||
size_t next_quote = line.find_first_of(my_quote, next_equals_sign + 2);
|
||||
if (next_quote == std::string::npos) { // no matching quote was found.
|
||||
KALDI_WARN << "No matching quote for " << my_quote << " in config line '"
|
||||
<< line << "'";
|
||||
return false;
|
||||
} else {
|
||||
std::string value(line, next_equals_sign + 2,
|
||||
next_quote - next_equals_sign - 2);
|
||||
data_.insert(std::make_pair(key, std::make_pair(value, false)));
|
||||
pos = next_quote + 1;
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
// we want to be able to parse something like "... input=Offset(a, -1) foo=bar":
|
||||
// in general, config values with spaces in them, even without quoting.
|
||||
|
||||
size_t next_next_equals_sign = line.find_first_of("=", next_equals_sign + 1),
|
||||
terminating_space = size;
|
||||
|
||||
if (next_next_equals_sign != std::string::npos) { // found a later equals sign.
|
||||
size_t preceding_space = line.find_last_of(" \t", next_next_equals_sign);
|
||||
if (preceding_space != std::string::npos &&
|
||||
preceding_space > next_equals_sign)
|
||||
terminating_space = preceding_space;
|
||||
}
|
||||
while (isspace(line[terminating_space - 1]) && terminating_space > 0)
|
||||
terminating_space--;
|
||||
|
||||
std::string value(line, next_equals_sign + 1,
|
||||
terminating_space - (next_equals_sign + 1));
|
||||
data_.insert(std::make_pair(key, std::make_pair(value, false)));
|
||||
pos = terminating_space;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ConfigLine::GetValue(const std::string &key, std::string *value) {
|
||||
KALDI_ASSERT(value != NULL);
|
||||
std::map<std::string, std::pair<std::string, bool> >::iterator it = data_.begin();
|
||||
for (; it != data_.end(); ++it) {
|
||||
if (it->first == key) {
|
||||
*value = (it->second).first;
|
||||
(it->second).second = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ConfigLine::GetValue(const std::string &key, BaseFloat *value) {
|
||||
KALDI_ASSERT(value != NULL);
|
||||
std::map<std::string, std::pair<std::string, bool> >::iterator it = data_.begin();
|
||||
for (; it != data_.end(); ++it) {
|
||||
if (it->first == key) {
|
||||
if (!ConvertStringToReal((it->second).first, value))
|
||||
return false;
|
||||
(it->second).second = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ConfigLine::GetValue(const std::string &key, int32 *value) {
|
||||
KALDI_ASSERT(value != NULL);
|
||||
std::map<std::string, std::pair<std::string, bool> >::iterator it = data_.begin();
|
||||
for (; it != data_.end(); ++it) {
|
||||
if (it->first == key) {
|
||||
if (!ConvertStringToInteger((it->second).first, value))
|
||||
return false;
|
||||
(it->second).second = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ConfigLine::GetValue(const std::string &key, std::vector<int32> *value) {
|
||||
KALDI_ASSERT(value != NULL);
|
||||
value->clear();
|
||||
std::map<std::string, std::pair<std::string, bool> >::iterator it = data_.begin();
|
||||
for (; it != data_.end(); ++it) {
|
||||
if (it->first == key) {
|
||||
if (!SplitStringToIntegers((it->second).first, ":,", true, value)) {
|
||||
// KALDI_WARN << "Bad option " << (it->second).first;
|
||||
return false;
|
||||
}
|
||||
(it->second).second = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ConfigLine::GetValue(const std::string &key, bool *value) {
|
||||
KALDI_ASSERT(value != NULL);
|
||||
std::map<std::string, std::pair<std::string, bool> >::iterator it = data_.begin();
|
||||
for (; it != data_.end(); ++it) {
|
||||
if (it->first == key) {
|
||||
if ((it->second).first.size() == 0) return false;
|
||||
switch (((it->second).first)[0]) {
|
||||
case 'F':
|
||||
case 'f':
|
||||
*value = false;
|
||||
break;
|
||||
case 'T':
|
||||
case 't':
|
||||
*value = true;
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
(it->second).second = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ConfigLine::HasUnusedValues() const {
|
||||
std::map<std::string, std::pair<std::string, bool> >::const_iterator it = data_.begin();
|
||||
for (; it != data_.end(); ++it) {
|
||||
if (!(it->second).second) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string ConfigLine::UnusedValues() const {
|
||||
std::string unused_str;
|
||||
std::map<std::string, std::pair<std::string, bool> >::const_iterator it = data_.begin();
|
||||
for (; it != data_.end(); ++it) {
|
||||
if (!(it->second).second) {
|
||||
if (unused_str == "")
|
||||
unused_str = it->first + "=" + (it->second).first;
|
||||
else
|
||||
unused_str += " " + it->first + "=" + (it->second).first;
|
||||
}
|
||||
}
|
||||
return unused_str;
|
||||
}
|
||||
|
||||
//// This is like ExpectToken but for two tokens, and it
|
||||
//// will either accept token1 and then token2, or just token2.
|
||||
//// This is useful in Read functions where the first token
|
||||
//// may already have been consumed.
|
||||
//void ExpectOneOrTwoTokens(std::istream &is, bool binary,
|
||||
// const std::string &token1,
|
||||
// const std::string &token2) {
|
||||
// KALDI_ASSERT(token1 != token2);
|
||||
// std::string temp;
|
||||
// ReadToken(is, binary, &temp);
|
||||
// if (temp == token1) {
|
||||
// ExpectToken(is, binary, token2);
|
||||
// } else {
|
||||
// if (temp != token2) {
|
||||
// KALDI_ERR << "Expecting token " << token1 << " or " << token2
|
||||
// << " but got " << temp;
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
|
||||
|
||||
bool IsValidName(const std::string &name) {
|
||||
if (name.size() == 0) return false;
|
||||
for (size_t i = 0; i < name.size(); i++) {
|
||||
if (i == 0 && !isalpha(name[i]) && name[i] != '_')
|
||||
return false;
|
||||
if (!isalnum(name[i]) && name[i] != '_' && name[i] != '-' && name[i] != '.')
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void ReadConfigLines(std::istream &is,
|
||||
std::vector<std::string> *lines) {
|
||||
KALDI_ASSERT(lines != NULL);
|
||||
std::string line;
|
||||
while (std::getline(is, line)) {
|
||||
if (line.size() == 0) continue;
|
||||
size_t start = line.find_first_not_of(" \t");
|
||||
size_t end = line.find_first_of('#');
|
||||
if (start == std::string::npos || start == end) continue;
|
||||
end = line.find_last_not_of(" \t", end - 1);
|
||||
KALDI_ASSERT(end >= start);
|
||||
lines->push_back(line.substr(start, end - start + 1));
|
||||
}
|
||||
}
|
||||
|
||||
void ParseConfigLines(const std::vector<std::string> &lines,
|
||||
std::vector<ConfigLine> *config_lines) {
|
||||
config_lines->resize(lines.size());
|
||||
for (size_t i = 0; i < lines.size(); i++) {
|
||||
bool ret = (*config_lines)[i].ParseLine(lines[i]);
|
||||
if (!ret) {
|
||||
KALDI_ERR << "Error parsing config line: " << lines[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // end namespace kaldi
|
||||
@@ -0,0 +1,281 @@
|
||||
// util/text-utils.h
|
||||
|
||||
// Copyright 2009-2011 Saarland University; Microsoft Corporation
|
||||
|
||||
// See ../../COPYING for clarification regarding multiple authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
// See the Apache 2 License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef KALDI_UTIL_TEXT_UTILS_H_
|
||||
#define KALDI_UTIL_TEXT_UTILS_H_
|
||||
|
||||
#include <errno.h>
|
||||
#include <string>
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
#include <limits>
|
||||
#include "base/kaldi-common.h"
|
||||
|
||||
|
||||
namespace kaldi {
|
||||
|
||||
/// Split a string using any of the single character delimiters.
|
||||
/// If omit_empty_strings == true, the output will contain any
|
||||
/// nonempty strings after splitting on any of the
|
||||
/// characters in the delimiter. If omit_empty_strings == false,
|
||||
/// the output will contain n+1 strings if there are n characters
|
||||
/// in the set "delim" within the input string. In this case
|
||||
/// the empty string is split to a single empty string.
|
||||
void SplitStringToVector(const std::string &full, const char *delim,
|
||||
bool omit_empty_strings,
|
||||
std::vector<std::string> *out);
|
||||
|
||||
/// Joins the elements of a vector of strings into a single string using
|
||||
/// "delim" as the delimiter. If omit_empty_strings == true, any empty strings
|
||||
/// in the vector are skipped. A vector of empty strings results in an empty
|
||||
/// string on the output.
|
||||
void JoinVectorToString(const std::vector<std::string> &vec_in,
|
||||
const char *delim, bool omit_empty_strings,
|
||||
std::string *str_out);
|
||||
|
||||
/**
|
||||
\brief Split a string (e.g. 1:2:3) into a vector of integers.
|
||||
|
||||
\param [in] delim String containing a list of characters, any of which
|
||||
is allowed as a delimiter.
|
||||
\param [in] omit_empty_strings If true, empty strings between delimiters are
|
||||
allowed and will not produce an output integer; if false,
|
||||
instances of characters in 'delim' that are consecutive or
|
||||
at the start or end of the string would be an error.
|
||||
You'll normally want this to be true if 'delim' consists
|
||||
of spaces, and false otherwise.
|
||||
\param [out] out The output list of integers.
|
||||
*/
|
||||
template<class I>
|
||||
bool SplitStringToIntegers(const std::string &full,
|
||||
const char *delim,
|
||||
bool omit_empty_strings, // typically false [but
|
||||
// should probably be true
|
||||
// if "delim" is spaces].
|
||||
std::vector<I> *out) {
|
||||
KALDI_ASSERT(out != NULL);
|
||||
KALDI_ASSERT_IS_INTEGER_TYPE(I);
|
||||
if (*(full.c_str()) == '\0') {
|
||||
out->clear();
|
||||
return true;
|
||||
}
|
||||
std::vector<std::string> split;
|
||||
SplitStringToVector(full, delim, omit_empty_strings, &split);
|
||||
out->resize(split.size());
|
||||
for (size_t i = 0; i < split.size(); i++) {
|
||||
const char *this_str = split[i].c_str();
|
||||
char *end = NULL;
|
||||
int64 j = 0;
|
||||
j = KALDI_STRTOLL(this_str, &end);
|
||||
if (end == this_str || *end != '\0') {
|
||||
out->clear();
|
||||
return false;
|
||||
} else {
|
||||
I jI = static_cast<I>(j);
|
||||
if (static_cast<int64>(jI) != j) {
|
||||
// output type cannot fit this integer.
|
||||
out->clear();
|
||||
return false;
|
||||
}
|
||||
(*out)[i] = jI;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// This is defined for F = float and double.
|
||||
template<class F>
|
||||
bool SplitStringToFloats(const std::string &full,
|
||||
const char *delim,
|
||||
bool omit_empty_strings, // typically false
|
||||
std::vector<F> *out);
|
||||
|
||||
|
||||
/// Converts a string into an integer via strtoll and returns false if there was
|
||||
/// any kind of problem (i.e. the string was not an integer or contained extra
|
||||
/// non-whitespace junk, or the integer was too large to fit into the type it is
|
||||
/// being converted into). Only sets *out if everything was OK and it returns
|
||||
/// true.
|
||||
template<class Int>
|
||||
bool ConvertStringToInteger(const std::string &str,
|
||||
Int *out) {
|
||||
KALDI_ASSERT_IS_INTEGER_TYPE(Int);
|
||||
const char *this_str = str.c_str();
|
||||
char *end = NULL;
|
||||
errno = 0;
|
||||
int64 i = KALDI_STRTOLL(this_str, &end);
|
||||
if (end != this_str)
|
||||
while (isspace(*end)) end++;
|
||||
if (end == this_str || *end != '\0' || errno != 0)
|
||||
return false;
|
||||
Int iInt = static_cast<Int>(i);
|
||||
if (static_cast<int64>(iInt) != i ||
|
||||
(i < 0 && !std::numeric_limits<Int>::is_signed)) {
|
||||
return false;
|
||||
}
|
||||
*out = iInt;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
/// ConvertStringToReal converts a string into either float or double
|
||||
/// and returns false if there was any kind of problem (i.e. the string
|
||||
/// was not a floating point number or contained extra non-whitespace junk).
|
||||
/// Be careful- this function will successfully read inf's or nan's.
|
||||
template <typename T>
|
||||
bool ConvertStringToReal(const std::string &str,
|
||||
T *out);
|
||||
|
||||
/// Removes the beginning and trailing whitespaces from a string
|
||||
void Trim(std::string *str);
|
||||
|
||||
|
||||
/// Removes leading and trailing white space from the string, then splits on the
|
||||
/// first section of whitespace found (if present), putting the part before the
|
||||
/// whitespace in "first" and the rest in "rest". If there is no such space,
|
||||
/// everything that remains after removing leading and trailing whitespace goes
|
||||
/// in "first".
|
||||
void SplitStringOnFirstSpace(const std::string &line,
|
||||
std::string *first,
|
||||
std::string *rest);
|
||||
|
||||
|
||||
/// Returns true if "token" is nonempty, and all characters are
|
||||
/// printable and whitespace-free.
|
||||
bool IsToken(const std::string &token);
|
||||
|
||||
|
||||
/// Returns true if "line" is free of \n characters and unprintable
|
||||
/// characters, and does not contain leading or trailing whitespace.
|
||||
bool IsLine(const std::string &line);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
This function returns true when two text strings are approximately equal, and
|
||||
false when they are not. The definition of 'equal' is normal string
|
||||
equality, except that two substrings like "0.31134" and "0.311341" would be
|
||||
considered equal. 'decimal_places_tolerance' controls how many digits after
|
||||
the '.' have to match up.
|
||||
E.g. StringsApproxEqual("hello 0.23 there", "hello 0.24 there", 2) would
|
||||
return false because there is a difference in the 2nd decimal, but with
|
||||
an argument of 1 it would return true.
|
||||
*/
|
||||
bool StringsApproxEqual(const std::string &a,
|
||||
const std::string &b,
|
||||
int32 decimal_places_check = 2);
|
||||
|
||||
/**
|
||||
This class is responsible for parsing input like
|
||||
hi-there xx=yyy a=b c empty= f-oo=Append(bar, sss) ba_z=123 bing='a b c' baz="a b c d='a b' e"
|
||||
and giving you access to the fields, in this case
|
||||
|
||||
FirstToken() == "hi-there", and key->value pairs:
|
||||
|
||||
xx->yyy, a->"b c", empty->"", f-oo->"Append(bar, sss)", ba_z->"123",
|
||||
bing->"a b c", baz->"a b c d='a b' e"
|
||||
|
||||
The first token is optional, if the line started with a key-value pair then
|
||||
FirstValue() will be empty.
|
||||
|
||||
Note: it can parse value fields with space inside them only if they are free of the '='
|
||||
character. If values are going to contain the '=' character, you need to quote them
|
||||
with either single or double quotes.
|
||||
|
||||
Key values may contain -_a-zA-Z0-9, but must begin with a-zA-Z_.
|
||||
*/
|
||||
class ConfigLine {
|
||||
public:
|
||||
// Tries to parse the line as a config-file line. Returns false
|
||||
// if it could not for some reason, e.g. parsing failure. In most cases
|
||||
// prints no warnings; the user should do this. Does not expect comments.
|
||||
bool ParseLine(const std::string &line);
|
||||
|
||||
// the GetValue functions are overloaded for various types. They return true
|
||||
// if the key exists with value that can be converted to that type, and false
|
||||
// otherwise. They also mark the key-value pair as having been read. It is
|
||||
// not an error to read values twice.
|
||||
bool GetValue(const std::string &key, std::string *value);
|
||||
bool GetValue(const std::string &key, BaseFloat *value);
|
||||
bool GetValue(const std::string &key, int32 *value);
|
||||
// Values may be separated by ":" or by ",".
|
||||
bool GetValue(const std::string &key, std::vector<int32> *value);
|
||||
bool GetValue(const std::string &key, bool *value);
|
||||
|
||||
bool HasUnusedValues() const;
|
||||
/// returns e.g. foo=bar xxx=yyy if foo and xxx were not consumed by one
|
||||
/// of the GetValue() functions.
|
||||
std::string UnusedValues() const;
|
||||
|
||||
const std::string &FirstToken() const { return first_token_; }
|
||||
|
||||
const std::string WholeLine() { return whole_line_; }
|
||||
// use default assignment operator and copy constructor.
|
||||
private:
|
||||
std::string whole_line_;
|
||||
// the first token of the line, e.g. if line is
|
||||
// foo-bar baz=bing
|
||||
// then first_token_ would be "foo-bar".
|
||||
std::string first_token_;
|
||||
|
||||
// data_ maps from key to (value, is-this-value-consumed?).
|
||||
std::map<std::string, std::pair<std::string, bool> > data_;
|
||||
|
||||
};
|
||||
|
||||
/// This function is like ExpectToken but for two tokens, and it will either
|
||||
/// accept token1 and then token2, or just token2. This is useful in Read
|
||||
/// functions where the first token may already have been consumed.
|
||||
void ExpectOneOrTwoTokens(std::istream &is, bool binary,
|
||||
const std::string &token1,
|
||||
const std::string &token2);
|
||||
|
||||
|
||||
/**
|
||||
This function reads in a config file and *appends* its contents to a vector of
|
||||
lines; it is responsible for removing comments (anything after '#') and
|
||||
stripping out any lines that contain only whitespace after comment removal.
|
||||
*/
|
||||
void ReadConfigLines(std::istream &is,
|
||||
std::vector<std::string> *lines);
|
||||
|
||||
|
||||
/**
|
||||
This function converts config-lines from a simple sequence of strings
|
||||
as output by ReadConfigLines(), into a sequence of first-tokens and
|
||||
name-value pairs. The general format is:
|
||||
"command-type bar=baz xx=yyy"
|
||||
etc., although there are subtleties as to what exactly is allowed, see
|
||||
documentation for class ConfigLine for details.
|
||||
This function will die if there was a parsing failure.
|
||||
*/
|
||||
void ParseConfigLines(const std::vector<std::string> &lines,
|
||||
std::vector<ConfigLine> *config_lines);
|
||||
|
||||
|
||||
/// Returns true if 'name' would be a valid name for a component or node in a
|
||||
/// nnet3Nnet. This is a nonempty string beginning with A-Za-z_, and containing only
|
||||
/// '-', '_', '.', A-Z, a-z, or 0-9.
|
||||
bool IsValidName(const std::string &name);
|
||||
|
||||
} // namespace kaldi
|
||||
|
||||
#endif // KALDI_UTIL_TEXT_UTILS_H_
|
||||
Reference in New Issue
Block a user