Initial commit: FunASR Speech Recognition Toolkit

Add complete FunASR codebase including models, runtime, and documentation.

runtime/onnxruntime/CMakeLists.txt

@@ -0,0 +1,95 @@
+cmake_minimum_required(VERSION 3.16)
+
+project(FunASROnnx)
+
+option(ENABLE_GLOG "Whether to build glog" ON)
+option(ENABLE_FST "Whether to build openfst" ON) # ITN need openfst compiled
+option(ENABLE_FFMPEG "Whether to enable ffmpeg audio decoding" OFF)
+option(GPU "Whether to build with GPU" OFF)
+
+# set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14 CACHE STRING "The C++ version to be used.")
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+
+include(TestBigEndian)
+test_big_endian(BIG_ENDIAN)
+if(BIG_ENDIAN)
+    message("Big endian system")
+else()
+    message("Little endian system")
+endif()
+
+# json
+include(FetchContent)
+if(NOT EXISTS ${PROJECT_SOURCE_DIR}/third_party/json/ChangeLog.md )
+FetchContent_Declare(json
+  URL   https://github.com/nlohmann/json/archive/refs/tags/v3.11.2.tar.gz
+SOURCE_DIR ${PROJECT_SOURCE_DIR}/third_party/json
+)
+
+FetchContent_MakeAvailable(json)
+endif()
+
+# for onnxruntime
+IF(WIN32)
+    file(REMOVE ${PROJECT_SOURCE_DIR}/third_party/glog/src/config.h 
+                ${PROJECT_SOURCE_DIR}/third_party/glog/src/glog/export.h 
+                ${PROJECT_SOURCE_DIR}/third_party/glog/src/glog/logging.h 
+                ${PROJECT_SOURCE_DIR}/third_party/glog/src/glog/raw_logging.h 
+                ${PROJECT_SOURCE_DIR}/third_party/glog/src/glog/stl_logging.h 
+                ${PROJECT_SOURCE_DIR}/third_party/glog/src/glog/vlog_is_on.h)
+ELSE()
+    link_directories(${ONNXRUNTIME_DIR}/lib)
+    if(ENABLE_FFMPEG AND NOT APPLE)
+        link_directories(${FFMPEG_DIR}/lib)
+    endif()
+endif()
+include_directories(${CMAKE_SOURCE_DIR}/src)
+include_directories(${PROJECT_SOURCE_DIR}/third_party/kaldi-native-fbank)
+include_directories(${PROJECT_SOURCE_DIR}/third_party/yaml-cpp/include)
+include_directories(${PROJECT_SOURCE_DIR}/third_party/jieba/include)
+include_directories(${PROJECT_SOURCE_DIR}/third_party/jieba/include/limonp/include)
+include_directories(${PROJECT_SOURCE_DIR}/third_party/kaldi)
+include_directories(${PROJECT_SOURCE_DIR}/third_party/json/include)
+
+if(GPU)
+    add_definitions(-DUSE_GPU)
+    set(TORCH_DIR "/usr/local/lib/python3.8/dist-packages/torch")
+    set(TORCH_BLADE_DIR "/usr/local/lib/python3.8/dist-packages/torch_blade")
+    include_directories(${TORCH_DIR}/include)
+    include_directories(${TORCH_DIR}/include/torch/csrc/api/include)
+    link_directories(${TORCH_DIR}/lib)
+    link_directories(${TORCH_BLADE_DIR})
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -D_GLIBCXX_USE_CXX11_ABI=0")
+endif()
+
+if(ENABLE_GLOG)
+    include_directories(${PROJECT_SOURCE_DIR}/third_party/glog/src)
+    set(BUILD_TESTING OFF)
+    add_subdirectory(third_party/glog)
+    include_directories(${glog_BINARY_DIR})
+    
+endif()
+
+if(ENABLE_FST)
+    # fst depend on glog and gflags
+    include_directories(${PROJECT_SOURCE_DIR}/third_party/gflags)
+    add_subdirectory(third_party/gflags)
+    include_directories(${gflags_BINARY_DIR}/include)
+    
+    # the following openfst if cloned from https://github.com/kkm000/openfst.git
+    # with some patch to fix the make errors. 
+    add_subdirectory(third_party/openfst)
+    include_directories(${openfst_SOURCE_DIR}/src/include)
+    if(WIN32)
+    include_directories(${openfst_SOURCE_DIR}/src/lib)
+    endif()
+
+endif()
+
+add_subdirectory(third_party/yaml-cpp)
+add_subdirectory(third_party/kaldi-native-fbank/kaldi-native-fbank/csrc)
+add_subdirectory(third_party/kaldi)
+add_subdirectory(src)
+add_subdirectory(bin)

runtime/onnxruntime/CMakeSettings.json

@@ -0,0 +1,44 @@
+{
+  "configurations": [
+    {
+      "name": "x64-Debug",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": ""
+    },
+    {
+      "name": "x64-Release",
+      "generator": "Ninja",
+      "configurationType": "RelWithDebInfo",
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "inheritEnvironments": [ "msvc_x64_x64" ]
+    },
+    {
+      "name": "Linux-GCC-Debug",
+      "generator": "Unix Makefiles",
+      "configurationType": "Debug",
+      "cmakeExecutable": "cmake",
+      "remoteCopySourcesExclusionList": [ ".vs", ".git", "out" ],
+      "cmakeCommandArgs": "-DONNXRUNTIME_DIR=/data/linux/thirdpart/onnxruntime-linux-x64-1.14.1",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "inheritEnvironments": [ "linux_x64" ],
+      "remoteMachineName": "${defaultRemoteMachineName}",
+      "remoteCMakeListsRoot": "$HOME/.vs/${projectDirName}/${workspaceHash}/src",
+      "remoteBuildRoot": "$HOME/.vs/${projectDirName}/${workspaceHash}/out/build/${name}",
+      "remoteInstallRoot": "$HOME/.vs/${projectDirName}/${workspaceHash}/out/install/${name}",
+      "remoteCopySources": true,
+      "rsyncCommandArgs": "-t --delete",
+      "remoteCopyBuildOutput": false,
+      "remoteCopySourcesMethod": "rsync"
+    }
+  ]
+}

runtime/onnxruntime/bin/CMakeLists.txt

@@ -0,0 +1,56 @@
+include_directories(${CMAKE_SOURCE_DIR}/include)
+
+if(WIN32)
+add_compile_options("$<$<CXX_COMPILER_ID:MSVC>:/execution-charset:utf-8>")
+add_compile_options("$<$<CXX_COMPILER_ID:MSVC>:/source-charset:utf-8>")
+include_directories(${ONNXRUNTIME_DIR}/include)
+if(ENABLE_FFMPEG)
+include_directories(${FFMPEG_DIR}/include)
+endif()
+include_directories(${PROJECT_SOURCE_DIR}/third_party)
+SET(RELATION_SOURCE "../src/resample.cpp" "../src/util.cpp" "../src/alignedmem.cpp" "../src/encode_converter.cpp")
+endif()
+
+add_executable(funasr-onnx-offline "funasr-onnx-offline.cpp" ${RELATION_SOURCE})
+target_link_options(funasr-onnx-offline PRIVATE "-Wl,--no-as-needed")
+target_link_libraries(funasr-onnx-offline PUBLIC funasr)
+
+add_executable(funasr-onnx-offline-vad "funasr-onnx-offline-vad.cpp" ${RELATION_SOURCE})
+target_link_options(funasr-onnx-offline-vad PRIVATE "-Wl,--no-as-needed")
+target_link_libraries(funasr-onnx-offline-vad PUBLIC funasr)
+
+add_executable(funasr-onnx-online-vad "funasr-onnx-online-vad.cpp" ${RELATION_SOURCE})
+target_link_options(funasr-onnx-online-vad PRIVATE "-Wl,--no-as-needed")
+target_link_libraries(funasr-onnx-online-vad PUBLIC funasr)
+
+add_executable(funasr-onnx-online-asr "funasr-onnx-online-asr.cpp" ${RELATION_SOURCE})
+target_link_options(funasr-onnx-online-asr PRIVATE "-Wl,--no-as-needed")
+target_link_libraries(funasr-onnx-online-asr PUBLIC funasr)
+
+add_executable(funasr-onnx-offline-punc "funasr-onnx-offline-punc.cpp" ${RELATION_SOURCE})
+target_link_options(funasr-onnx-offline-punc PRIVATE "-Wl,--no-as-needed")
+target_link_libraries(funasr-onnx-offline-punc PUBLIC funasr)
+
+add_executable(funasr-onnx-online-punc "funasr-onnx-online-punc.cpp" ${RELATION_SOURCE})
+target_link_options(funasr-onnx-online-punc PRIVATE "-Wl,--no-as-needed")
+target_link_libraries(funasr-onnx-online-punc PUBLIC funasr)
+
+add_executable(funasr-onnx-offline-rtf "funasr-onnx-offline-rtf.cpp" ${RELATION_SOURCE})
+target_link_options(funasr-onnx-offline-rtf PRIVATE "-Wl,--no-as-needed")
+target_link_libraries(funasr-onnx-offline-rtf PUBLIC funasr)
+
+add_executable(funasr-onnx-2pass "funasr-onnx-2pass.cpp" ${RELATION_SOURCE})
+target_link_options(funasr-onnx-2pass PRIVATE "-Wl,--no-as-needed")
+target_link_libraries(funasr-onnx-2pass PUBLIC funasr)
+
+add_executable(funasr-onnx-2pass-rtf "funasr-onnx-2pass-rtf.cpp" ${RELATION_SOURCE})
+target_link_options(funasr-onnx-2pass-rtf PRIVATE "-Wl,--no-as-needed")
+target_link_libraries(funasr-onnx-2pass-rtf PUBLIC funasr)
+
+add_executable(funasr-onnx-online-rtf "funasr-onnx-online-rtf.cpp" ${RELATION_SOURCE})
+target_link_options(funasr-onnx-online-rtf PRIVATE "-Wl,--no-as-needed")
+target_link_libraries(funasr-onnx-online-rtf PUBLIC funasr)
+
+# include_directories(${FFMPEG_DIR}/include)
+# add_executable(ff "ffmpeg.cpp")
+# target_link_libraries(ff PUBLIC avutil avcodec avformat swresample)

runtime/onnxruntime/bin/funasr-onnx-2pass-rtf.cpp

@@ -0,0 +1,369 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#ifndef _WIN32
+#include <sys/time.h>
+#else
+#include <win_func.h>
+#endif
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <map>
+#include <atomic>
+#include <mutex>
+#include <thread>
+#include <glog/logging.h>
+#include "util.h"
+#include "funasrruntime.h"
+#include "tclap/CmdLine.h"
+#include "com-define.h"
+#include "audio.h"
+
+using namespace std;
+
+std::atomic<int> wav_index(0);
+std::mutex mtx;
+
+bool is_target_file(const std::string& filename, const std::string target) {
+    std::size_t pos = filename.find_last_of(".");
+    if (pos == std::string::npos) {
+        return false;
+    }
+    std::string extension = filename.substr(pos + 1);
+    return (extension == target);
+}
+
+void GetValue(TCLAP::ValueArg<std::string>& value_arg, string key, std::map<std::string, std::string>& model_path)
+{
+    model_path.insert({key, value_arg.getValue()});
+    LOG(INFO)<< key << " : " << value_arg.getValue();
+}
+
+void runReg(FUNASR_HANDLE tpass_handle, std::vector<int> chunk_size, vector<string> wav_list, vector<string> wav_ids, int audio_fs,
+            float* total_length, long* total_time, int core_id, ASR_TYPE asr_mode_, string nn_hotwords_,
+            float glob_beam, float lat_beam, float am_scale, int inc_bias, unordered_map<string, int> hws_map) {
+    
+    struct timeval start, end;
+    long seconds = 0;
+    float n_total_length = 0.0f;
+    long n_total_time = 0;
+
+    FUNASR_DEC_HANDLE decoder_handle = FunASRWfstDecoderInit(tpass_handle, ASR_TWO_PASS, glob_beam, lat_beam, am_scale);
+    // load hotwords list and build graph
+    FunWfstDecoderLoadHwsRes(decoder_handle, inc_bias, hws_map);
+       
+    std::vector<std::vector<float>> hotwords_embedding = CompileHotwordEmbedding(tpass_handle, nn_hotwords_, ASR_TWO_PASS);
+    
+    // init online features
+    FUNASR_HANDLE tpass_online_handle=FunTpassOnlineInit(tpass_handle, chunk_size);
+
+    // warm up
+    for (size_t i = 0; i < 2; i++)
+    {
+        int32_t sampling_rate_ = audio_fs;
+        funasr::Audio audio(1);
+		if(is_target_file(wav_list[0].c_str(), "wav")){
+			if(!audio.LoadWav2Char(wav_list[0].c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_list[0];
+                exit(-1);
+            }
+		}else if(is_target_file(wav_list[0].c_str(), "pcm")){
+			if (!audio.LoadPcmwav2Char(wav_list[0].c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_list[0];
+                exit(-1);
+            }
+		}else{
+			if (!audio.FfmpegLoad(wav_list[0].c_str(), true)){
+				LOG(ERROR)<<"Failed to load "<< wav_list[0];
+                exit(-1);
+            }
+		}
+        char* speech_buff = audio.GetSpeechChar();
+        int buff_len = audio.GetSpeechLen()*2;
+
+        int step = 1600*2;
+        bool is_final = false;
+        std::vector<std::vector<string>> punc_cache(2);
+        for (int sample_offset = 0; sample_offset < buff_len; sample_offset += std::min(step, buff_len - sample_offset)) {
+            if (sample_offset + step >= buff_len - 1) {
+                    step = buff_len - sample_offset;
+                    is_final = true;
+                } else {
+                    is_final = false;
+            }
+            FUNASR_RESULT result = FunTpassInferBuffer(tpass_handle, tpass_online_handle, speech_buff+sample_offset, step, punc_cache, is_final, 
+                                                        sampling_rate_, "pcm", (ASR_TYPE)asr_mode_, hotwords_embedding, true, decoder_handle);
+            if (result)
+            {
+                FunASRFreeResult(result);
+            }
+        }
+    }
+
+    while (true) {
+        // 使用原子变量获取索引并递增
+        int i = wav_index.fetch_add(1);
+        if (i >= wav_list.size()) {
+            break;
+        }
+        int32_t sampling_rate_ = audio_fs;
+        funasr::Audio audio(1);
+		if(is_target_file(wav_list[i].c_str(), "wav")){
+			if(!audio.LoadWav2Char(wav_list[i].c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_list[i];
+                exit(-1);
+            }
+		}else if(is_target_file(wav_list[i].c_str(), "pcm")){
+			if (!audio.LoadPcmwav2Char(wav_list[i].c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_list[i];
+                exit(-1);
+            }
+		}else{
+			if (!audio.FfmpegLoad(wav_list[i].c_str(), true)){
+				LOG(ERROR)<<"Failed to load "<< wav_list[i];
+                exit(-1);
+            }
+		}
+        char* speech_buff = audio.GetSpeechChar();
+        int buff_len = audio.GetSpeechLen()*2;
+
+        int step = 1600*2;
+        bool is_final = false;
+        string online_res="";
+        string tpass_res="";
+        string time_stamp_res="";
+        std::vector<std::vector<string>> punc_cache(2);
+        for (int sample_offset = 0; sample_offset < buff_len; sample_offset += std::min(step, buff_len - sample_offset)) {
+            if (sample_offset + step >= buff_len - 1) {
+                    step = buff_len - sample_offset;
+                    is_final = true;
+                } else {
+                    is_final = false;
+            }
+            gettimeofday(&start, nullptr);
+            FUNASR_RESULT result = FunTpassInferBuffer(tpass_handle, tpass_online_handle, speech_buff+sample_offset, step, punc_cache, is_final, 
+                                                        sampling_rate_, "pcm", (ASR_TYPE)asr_mode_, hotwords_embedding, true, decoder_handle);
+            gettimeofday(&end, nullptr);
+            seconds = (end.tv_sec - start.tv_sec);
+            long taking_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+            n_total_time += taking_micros;
+
+            if (result)
+            {
+                string online_msg = FunASRGetResult(result, 0);
+                online_res += online_msg;
+                if(online_msg != ""){
+                    LOG(INFO) <<"Thread: " << this_thread::get_id() <<" " << wav_ids[i] <<" : "<<online_msg;
+                }
+                string tpass_msg = FunASRGetTpassResult(result, 0);
+                tpass_res += tpass_msg;
+                if(tpass_msg != ""){
+                    LOG(INFO) <<"Thread: " << this_thread::get_id() <<" " << wav_ids[i] <<" offline results : "<<tpass_msg;
+                }
+                string stamp = FunASRGetStamp(result);
+                if(stamp !=""){
+                    LOG(INFO) <<"Thread: " << this_thread::get_id() <<" " << wav_ids[i] << " time stamp : " << stamp;
+                    if(time_stamp_res == ""){
+                        time_stamp_res += stamp;
+                    }else{
+                        time_stamp_res = time_stamp_res.erase(time_stamp_res.length()-1)
+                                         + "," + stamp.substr(1);
+                    }
+                }
+                float snippet_time = FunASRGetRetSnippetTime(result);
+                n_total_length += snippet_time;
+                FunASRFreeResult(result);
+            }
+            else
+            {
+                LOG(ERROR) << ("No return data!\n");
+            }
+        }
+        if(asr_mode_ == 2){
+            LOG(INFO) <<"Thread: " << this_thread::get_id() <<" " << wav_ids[i] << " Final online  results "<<" : "<<online_res;
+        }
+        if(asr_mode_==1){
+            LOG(INFO) <<"Thread: " << this_thread::get_id() <<" " << wav_ids[i] << " Final online  results "<<" : "<<tpass_res;
+        }
+        if(asr_mode_ == 0 || asr_mode_==2){
+            LOG(INFO) <<"Thread: " << this_thread::get_id() <<" " << wav_ids[i] << " Final offline results " <<" : "<<tpass_res;
+            if(time_stamp_res!=""){
+                LOG(INFO) <<"Thread: " << this_thread::get_id() <<" " << wav_ids[i] << " Final timestamp results " <<" : "<<time_stamp_res;
+            }
+        }
+
+    }
+    {
+        lock_guard<mutex> guard(mtx);
+        *total_length += n_total_length;
+        if(*total_time < n_total_time){
+            *total_time = n_total_time;
+        }
+    }
+    FunWfstDecoderUnloadHwsRes(decoder_handle);
+    FunASRWfstDecoderUninit(decoder_handle);
+    FunTpassOnlineUninit(tpass_online_handle);
+}
+
+
+int main(int argc, char** argv)
+{
+    google::InitGoogleLogging(argv[0]);
+    FLAGS_logtostderr = true;
+
+    TCLAP::CmdLine cmd("funasr-onnx-2pass", ' ', "1.0");
+    TCLAP::ValueArg<std::string>    offline_model_dir("", OFFLINE_MODEL_DIR, "the asr offline model path, which contains model.onnx, config.yaml, am.mvn", true, "", "string");
+    TCLAP::ValueArg<std::string>    online_model_dir("", ONLINE_MODEL_DIR, "the asr online model path, which contains model.onnx, decoder.onnx, config.yaml, am.mvn", true, "", "string");
+    TCLAP::ValueArg<std::string>    quantize("", QUANTIZE, "true (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    vad_dir("", VAD_DIR, "the vad online model path, which contains model.onnx, vad.yaml, vad.mvn", false, "", "string");
+    TCLAP::ValueArg<std::string>    vad_quant("", VAD_QUANT, "true (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    punc_dir("", PUNC_DIR, "the punc online model path, which contains model.onnx, punc.yaml", false, "", "string");
+    TCLAP::ValueArg<std::string>    punc_quant("", PUNC_QUANT, "true (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    itn_dir("", ITN_DIR, "the itn model(fst) path, which contains zh_itn_tagger.fst and zh_itn_verbalizer.fst", false, "", "string");
+    TCLAP::ValueArg<std::string>    lm_dir("", LM_DIR, "the lm model path, which contains compiled models: TLG.fst, config.yaml, lexicon.txt ", false, "", "string");
+    TCLAP::ValueArg<float>    global_beam("", GLOB_BEAM, "the decoding beam for beam searching ", false, 3.0, "float");
+    TCLAP::ValueArg<float>    lattice_beam("", LAT_BEAM, "the lattice generation beam for beam searching ", false, 3.0, "float");
+    TCLAP::ValueArg<float>    am_scale("", AM_SCALE, "the acoustic scale for beam searching ", false, 10.0, "float");
+    TCLAP::ValueArg<std::int32_t>   fst_inc_wts("", FST_INC_WTS, "the fst hotwords incremental bias", false, 20, "int32_t");
+
+    TCLAP::ValueArg<std::string>    asr_mode("", ASR_MODE, "offline, online, 2pass", false, "2pass", "string");
+    TCLAP::ValueArg<std::int32_t>   onnx_thread("", "model-thread-num", "onnxruntime SetIntraOpNumThreads", false, 1, "int32_t");
+    TCLAP::ValueArg<std::int32_t>   thread_num_("", THREAD_NUM, "multi-thread num for rtf", false, 1, "int32_t");
+    TCLAP::ValueArg<std::string>    wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string");
+    TCLAP::ValueArg<std::int32_t>   audio_fs("", AUDIO_FS, "the sample rate of audio", false, 16000, "int32_t");
+    TCLAP::ValueArg<std::string>    hotword("", HOTWORD, "the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)", false, "", "string");
+
+    cmd.add(offline_model_dir);
+    cmd.add(online_model_dir);
+    cmd.add(quantize);
+    cmd.add(vad_dir);
+    cmd.add(vad_quant);
+    cmd.add(punc_dir);
+    cmd.add(punc_quant);
+    cmd.add(itn_dir);
+    cmd.add(lm_dir);
+    cmd.add(global_beam);
+    cmd.add(lattice_beam);
+    cmd.add(am_scale);
+    cmd.add(fst_inc_wts);
+    cmd.add(wav_path);
+    cmd.add(audio_fs);
+    cmd.add(asr_mode);
+    cmd.add(onnx_thread);
+    cmd.add(thread_num_);
+    cmd.add(hotword);
+    cmd.parse(argc, argv);
+
+    std::map<std::string, std::string> model_path;
+    GetValue(offline_model_dir, OFFLINE_MODEL_DIR, model_path);
+    GetValue(online_model_dir, ONLINE_MODEL_DIR, model_path);
+    GetValue(quantize, QUANTIZE, model_path);
+    GetValue(vad_dir, VAD_DIR, model_path);
+    GetValue(vad_quant, VAD_QUANT, model_path);
+    GetValue(punc_dir, PUNC_DIR, model_path);
+    GetValue(punc_quant, PUNC_QUANT, model_path);
+    GetValue(itn_dir, ITN_DIR, model_path);
+    GetValue(lm_dir, LM_DIR, model_path);
+    GetValue(wav_path, WAV_PATH, model_path);
+    GetValue(asr_mode, ASR_MODE, model_path);
+
+    struct timeval start, end;
+    gettimeofday(&start, nullptr);
+    int thread_num = onnx_thread.getValue();
+    int asr_mode_ = -1;
+    if(model_path[ASR_MODE] == "offline"){
+        asr_mode_ = 0;
+    }else if(model_path[ASR_MODE] == "online"){
+        asr_mode_ = 1;
+    }else if(model_path[ASR_MODE] == "2pass"){
+        asr_mode_ = 2;
+    }else{
+        LOG(ERROR) << "Wrong asr-mode : " << model_path[ASR_MODE];
+        exit(-1);
+    }
+    FUNASR_HANDLE tpass_hanlde=FunTpassInit(model_path, thread_num);
+
+    if (!tpass_hanlde)
+    {
+        LOG(ERROR) << "FunTpassInit init failed";
+        exit(-1);
+    }
+    float glob_beam = 3.0f;
+    float lat_beam = 3.0f;
+    float am_sc = 10.0f;
+    if (lm_dir.isSet()) {
+        glob_beam = global_beam.getValue();
+        lat_beam = lattice_beam.getValue();
+        am_sc = am_scale.getValue();
+    }
+
+    gettimeofday(&end, nullptr);
+    long seconds = (end.tv_sec - start.tv_sec);
+    long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+    LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s";
+
+    // hotword file
+    unordered_map<string, int> hws_map;
+    std::string nn_hotwords_ = "";
+    std::string hotword_path = hotword.getValue();
+    LOG(INFO) << "hotword path: " << hotword_path;
+    funasr::ExtractHws(hotword_path, hws_map, nn_hotwords_);
+
+    // read wav_path
+    vector<string> wav_list;
+    vector<string> wav_ids;
+    string default_id = "wav_default_id";
+    string wav_path_ = model_path.at(WAV_PATH);
+
+    if(is_target_file(wav_path_, "scp")){
+        ifstream in(wav_path_);
+        if (!in.is_open()) {
+            LOG(ERROR) << "Failed to open file: " << model_path.at(WAV_SCP) ;
+            return 0;
+        }
+        string line;
+        while(getline(in, line))
+        {
+            istringstream iss(line);
+            string column1, column2;
+            iss >> column1 >> column2;
+            wav_list.emplace_back(column2);
+            wav_ids.emplace_back(column1);
+        }
+        in.close();
+    }else{
+        wav_list.emplace_back(wav_path_);
+        wav_ids.emplace_back(default_id);
+    }
+
+    std::vector<int> chunk_size = {5,10,5};
+    // 多线程测试
+    float total_length = 0.0f;
+    long total_time = 0;
+    std::vector<std::thread> threads;
+
+    int rtf_threds = thread_num_.getValue();
+    for (int i = 0; i < rtf_threds; i++)
+    {
+        threads.emplace_back(thread(runReg, tpass_hanlde, chunk_size, wav_list, wav_ids, audio_fs.getValue(), &total_length, &total_time, i, (ASR_TYPE)asr_mode_, nn_hotwords_,
+                                    glob_beam, lat_beam, am_sc, fst_inc_wts.getValue(), hws_map));
+    }
+
+    for (auto& thread : threads)
+    {
+        thread.join();
+    }
+
+    LOG(INFO) << "total_time_wav " << (long)(total_length * 1000) << " ms";
+    LOG(INFO) << "total_time_comput " << total_time / 1000 << " ms";
+    LOG(INFO) << "total_rtf " << (double)total_time/ (total_length*1000000);
+    LOG(INFO) << "speedup " << 1.0/((double)total_time/ (total_length*1000000));
+
+    FunTpassUninit(tpass_hanlde);
+    return 0;
+}
+

runtime/onnxruntime/bin/funasr-onnx-2pass.cpp

@@ -0,0 +1,274 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#ifndef _WIN32
+#include <sys/time.h>
+#else
+#include <win_func.h>
+#endif
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <map>
+#include <glog/logging.h>
+#include "util.h"
+#include "funasrruntime.h"
+#include "tclap/CmdLine.h"
+#include "com-define.h"
+#include "audio.h"
+
+using namespace std;
+
+bool is_target_file(const std::string& filename, const std::string target) {
+    std::size_t pos = filename.find_last_of(".");
+    if (pos == std::string::npos) {
+        return false;
+    }
+    std::string extension = filename.substr(pos + 1);
+    return (extension == target);
+}
+
+void GetValue(TCLAP::ValueArg<std::string>& value_arg, string key, std::map<std::string, std::string>& model_path)
+{
+    model_path.insert({key, value_arg.getValue()});
+    LOG(INFO)<< key << " : " << value_arg.getValue();
+}
+
+int main(int argc, char** argv)
+{
+    google::InitGoogleLogging(argv[0]);
+    FLAGS_logtostderr = true;
+
+    TCLAP::CmdLine cmd("funasr-onnx-2pass", ' ', "1.0");
+    TCLAP::ValueArg<std::string>    offline_model_dir("", OFFLINE_MODEL_DIR, "the asr offline model path, which contains model.onnx, config.yaml, am.mvn", true, "", "string");
+    TCLAP::ValueArg<std::string>    online_model_dir("", ONLINE_MODEL_DIR, "the asr online model path, which contains model.onnx, decoder.onnx, config.yaml, am.mvn", true, "", "string");
+    TCLAP::ValueArg<std::string>    quantize("", QUANTIZE, "true (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    vad_dir("", VAD_DIR, "the vad online model path, which contains model.onnx, vad.yaml, vad.mvn", false, "", "string");
+    TCLAP::ValueArg<std::string>    vad_quant("", VAD_QUANT, "true (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    punc_dir("", PUNC_DIR, "the punc online model path, which contains model.onnx, punc.yaml", false, "", "string");
+    TCLAP::ValueArg<std::string>    punc_quant("", PUNC_QUANT, "true (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    itn_dir("", ITN_DIR, "the itn model(fst) path, which contains zh_itn_tagger.fst and zh_itn_verbalizer.fst", false, "", "string");
+    TCLAP::ValueArg<std::string>    lm_dir("", LM_DIR, "the lm model path, which contains compiled models: TLG.fst, config.yaml, lexicon.txt ", false, "", "string");
+    TCLAP::ValueArg<float>    global_beam("", GLOB_BEAM, "the decoding beam for beam searching ", false, 3.0, "float");
+    TCLAP::ValueArg<float>    lattice_beam("", LAT_BEAM, "the lattice generation beam for beam searching ", false, 3.0, "float");
+    TCLAP::ValueArg<float>    am_scale("", AM_SCALE, "the acoustic scale for beam searching ", false, 10.0, "float");
+    TCLAP::ValueArg<std::int32_t>   fst_inc_wts("", FST_INC_WTS, "the fst hotwords incremental bias", false, 20, "int32_t");
+    TCLAP::ValueArg<std::string>    asr_mode("", ASR_MODE, "offline, online, 2pass", false, "2pass", "string");
+    TCLAP::ValueArg<std::int32_t>   onnx_thread("", "model-thread-num", "onnxruntime SetIntraOpNumThreads", false, 1, "int32_t");
+
+    TCLAP::ValueArg<std::string> wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string");
+    TCLAP::ValueArg<std::int32_t>   audio_fs("", AUDIO_FS, "the sample rate of audio", false, 16000, "int32_t");
+    TCLAP::ValueArg<std::string>    hotword("", HOTWORD, "the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)", false, "", "string");
+
+    cmd.add(offline_model_dir);
+    cmd.add(online_model_dir);
+    cmd.add(quantize);
+    cmd.add(vad_dir);
+    cmd.add(vad_quant);
+    cmd.add(punc_dir);
+    cmd.add(punc_quant);
+    cmd.add(lm_dir);
+    cmd.add(global_beam);
+    cmd.add(lattice_beam);
+    cmd.add(am_scale);
+    cmd.add(fst_inc_wts);
+    cmd.add(itn_dir);
+    cmd.add(wav_path);
+    cmd.add(audio_fs);
+    cmd.add(asr_mode);
+    cmd.add(onnx_thread);
+    cmd.add(hotword);
+    cmd.parse(argc, argv);
+
+    std::map<std::string, std::string> model_path;
+    GetValue(offline_model_dir, OFFLINE_MODEL_DIR, model_path);
+    GetValue(online_model_dir, ONLINE_MODEL_DIR, model_path);
+    GetValue(quantize, QUANTIZE, model_path);
+    GetValue(vad_dir, VAD_DIR, model_path);
+    GetValue(vad_quant, VAD_QUANT, model_path);
+    GetValue(punc_dir, PUNC_DIR, model_path);
+    GetValue(punc_quant, PUNC_QUANT, model_path);
+    GetValue(lm_dir, LM_DIR, model_path);
+    GetValue(itn_dir, ITN_DIR, model_path);
+    GetValue(wav_path, WAV_PATH, model_path);
+    GetValue(asr_mode, ASR_MODE, model_path);
+
+    struct timeval start, end;
+    gettimeofday(&start, nullptr);
+    int thread_num = onnx_thread.getValue();
+    int asr_mode_ = -1;
+    if(model_path[ASR_MODE] == "offline"){
+        asr_mode_ = 0;
+    }else if(model_path[ASR_MODE] == "online"){
+        asr_mode_ = 1;
+    }else if(model_path[ASR_MODE] == "2pass"){
+        asr_mode_ = 2;
+    }else{
+        LOG(ERROR) << "Wrong asr-mode : " << model_path[ASR_MODE];
+        exit(-1);
+    }
+    FUNASR_HANDLE tpass_handle=FunTpassInit(model_path, thread_num);
+
+    if (!tpass_handle)
+    {
+        LOG(ERROR) << "FunTpassInit init failed";
+        exit(-1);
+    }
+    float glob_beam = 3.0f;
+    float lat_beam = 3.0f;
+    float am_sc = 10.0f;
+    if (lm_dir.isSet()) {
+        glob_beam = global_beam.getValue();
+        lat_beam = lattice_beam.getValue();
+        am_sc = am_scale.getValue();
+    }
+    // init wfst decoder
+    FUNASR_DEC_HANDLE decoder_handle = FunASRWfstDecoderInit(tpass_handle, ASR_TWO_PASS, glob_beam, lat_beam, am_sc);
+
+    gettimeofday(&end, nullptr);
+    long seconds = (end.tv_sec - start.tv_sec);
+    long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+    LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s";
+
+    // hotword file
+    unordered_map<string, int> hws_map;
+    std::string nn_hotwords_ = "";
+    std::string hotword_path = hotword.getValue();
+    LOG(INFO) << "hotword path: " << hotword_path;
+    funasr::ExtractHws(hotword_path, hws_map, nn_hotwords_);
+
+    // read wav_path
+    vector<string> wav_list;
+    vector<string> wav_ids;
+    string default_id = "wav_default_id";
+    string wav_path_ = model_path.at(WAV_PATH);
+
+    if(is_target_file(wav_path_, "scp")){
+        ifstream in(wav_path_);
+        if (!in.is_open()) {
+            LOG(ERROR) << "Failed to open file: " << model_path.at(WAV_SCP) ;
+            return 0;
+        }
+        string line;
+        while(getline(in, line))
+        {
+            istringstream iss(line);
+            string column1, column2;
+            iss >> column1 >> column2;
+            wav_list.emplace_back(column2);
+            wav_ids.emplace_back(column1);
+        }
+        in.close();
+    }else{
+        wav_list.emplace_back(wav_path_);
+        wav_ids.emplace_back(default_id);
+    }
+
+    // load hotwords list and build graph
+    FunWfstDecoderLoadHwsRes(decoder_handle, fst_inc_wts.getValue(), hws_map);
+
+    std::vector<std::vector<float>> hotwords_embedding = CompileHotwordEmbedding(tpass_handle, nn_hotwords_, ASR_TWO_PASS);
+    // init online features
+    std::vector<int> chunk_size = {5,10,5};
+    FUNASR_HANDLE tpass_online_handle=FunTpassOnlineInit(tpass_handle, chunk_size);
+    float snippet_time = 0.0f;
+    long taking_micros = 0;
+    for (int i = 0; i < wav_list.size(); i++) {
+        auto& wav_file = wav_list[i];
+        auto& wav_id = wav_ids[i];
+
+        int32_t sampling_rate_ = audio_fs.getValue();
+        funasr::Audio audio(1);
+		if(is_target_file(wav_file.c_str(), "wav")){
+			if(!audio.LoadWav2Char(wav_file.c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_file;
+                exit(-1);
+            }
+		}else if(is_target_file(wav_file.c_str(), "pcm")){
+			if (!audio.LoadPcmwav2Char(wav_file.c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_file;
+                exit(-1);
+            }
+		}else{
+			if (!audio.FfmpegLoad(wav_file.c_str(), true)){
+				LOG(ERROR)<<"Failed to load "<< wav_file;
+                exit(-1);
+            }
+		}
+        char* speech_buff = audio.GetSpeechChar();
+        int buff_len = audio.GetSpeechLen()*2;
+
+        int step = 800*2;
+        bool is_final = false;
+        string online_res="";
+        string tpass_res="";
+        string time_stamp_res="";
+        std::vector<std::vector<string>> punc_cache(2);
+        for (int sample_offset = 0; sample_offset < buff_len; sample_offset += std::min(step, buff_len - sample_offset)) {
+            if (sample_offset + step >= buff_len - 1) {
+                    step = buff_len - sample_offset;
+                    is_final = true;
+                } else {
+                    is_final = false;
+            }
+            gettimeofday(&start, nullptr);
+            FUNASR_RESULT result = FunTpassInferBuffer(tpass_handle, tpass_online_handle, 
+                speech_buff+sample_offset, step, punc_cache, is_final, sampling_rate_, "pcm", 
+                (ASR_TYPE)asr_mode_, hotwords_embedding, true, decoder_handle);
+            gettimeofday(&end, nullptr);
+            seconds = (end.tv_sec - start.tv_sec);
+            taking_micros += ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+
+            if (result)
+            {
+                string online_msg = FunASRGetResult(result, 0);
+                online_res += online_msg;
+                if(online_msg != ""){
+                    LOG(INFO)<< wav_id <<" : "<<online_msg;
+                }
+                string tpass_msg = FunASRGetTpassResult(result, 0);
+                tpass_res += tpass_msg;
+                if(tpass_msg != ""){
+                    LOG(INFO)<< wav_id <<" offline results : "<<tpass_msg;
+                }
+                string stamp = FunASRGetStamp(result);
+                if(stamp !=""){
+                    LOG(INFO) << wav_ids[i] << " time stamp : " << stamp;
+                    if(time_stamp_res == ""){
+                        time_stamp_res += stamp;
+                    }else{
+                        time_stamp_res = time_stamp_res.erase(time_stamp_res.length()-1)
+                                         + "," + stamp.substr(1);
+                    }
+                }
+                snippet_time += FunASRGetRetSnippetTime(result);
+                FunASRFreeResult(result);
+            }
+        }
+        if(asr_mode_==2){
+            LOG(INFO) << wav_id << " Final online  results "<<" : "<<online_res;
+        }
+        if(asr_mode_==1){
+            LOG(INFO) << wav_id << " Final online  results "<<" : "<<tpass_res;
+        }
+        if(asr_mode_==0 || asr_mode_==2){
+            LOG(INFO) << wav_id << " Final offline results " <<" : "<<tpass_res;
+            if(time_stamp_res!=""){
+                LOG(INFO) << wav_id << " Final timestamp results " <<" : "<<time_stamp_res;
+            }
+        }
+    }
+
+    FunWfstDecoderUnloadHwsRes(decoder_handle);
+    LOG(INFO) << "Audio length: " << (double)snippet_time << " s";
+    LOG(INFO) << "Model inference takes: " << (double)taking_micros / 1000000 <<" s";
+    LOG(INFO) << "Model inference RTF: " << (double)taking_micros/ (snippet_time*1000000);
+    FunASRWfstDecoderUninit(decoder_handle);
+    FunTpassOnlineUninit(tpass_online_handle);
+    FunTpassUninit(tpass_handle);
+    return 0;
+}
+

runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp

@@ -0,0 +1,100 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#ifndef _WIN32
+#include <sys/time.h>
+#else
+#include <win_func.h>
+#endif
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <map>
+#include <glog/logging.h>
+#include "funasrruntime.h"
+#include "tclap/CmdLine.h"
+#include "com-define.h"
+
+using namespace std;
+
+void GetValue(TCLAP::ValueArg<std::string>& value_arg, string key, std::map<std::string, std::string>& model_path)
+{
+    if (value_arg.isSet()){
+        model_path.insert({key, value_arg.getValue()});
+        LOG(INFO)<< key << " : " << value_arg.getValue();
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    google::InitGoogleLogging(argv[0]);
+    FLAGS_logtostderr = true;
+
+    TCLAP::CmdLine cmd("funasr-onnx-offline-punc", ' ', "1.0");
+    TCLAP::ValueArg<std::string>    model_dir("", MODEL_DIR, "the punc model path, which contains model.onnx, punc.yaml", true, "", "string");
+    TCLAP::ValueArg<std::string>    quantize("", QUANTIZE, "false (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string> txt_path("", TXT_PATH, "txt file path, one sentence per line", true, "", "string");
+
+    cmd.add(model_dir);
+    cmd.add(quantize);
+    cmd.add(txt_path);
+    cmd.parse(argc, argv);
+
+    std::map<std::string, std::string> model_path;
+    GetValue(model_dir, MODEL_DIR, model_path);
+    GetValue(quantize, QUANTIZE, model_path);
+    GetValue(txt_path, TXT_PATH, model_path);
+
+    struct timeval start, end;
+    gettimeofday(&start, nullptr);
+    int thread_num = 1;
+    FUNASR_HANDLE punc_hanlde=CTTransformerInit(model_path, thread_num);
+
+    if (!punc_hanlde)
+    {
+        LOG(ERROR) << "FunASR init failed";
+        exit(-1);
+    }
+
+    gettimeofday(&end, nullptr);
+    long seconds = (end.tv_sec - start.tv_sec);
+    long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+    LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s";
+
+    // read txt_path
+    vector<string> txt_list;
+
+    if(model_path.find(TXT_PATH)!=model_path.end()){
+        ifstream in(model_path.at(TXT_PATH));
+        if (!in.is_open()) {
+            LOG(ERROR) << "Failed to open file: " << model_path.at(TXT_PATH) ;
+            return 0;
+        }
+        string line;
+        while(getline(in, line))
+        {
+            txt_list.emplace_back(line); 
+        }
+        in.close();
+    }
+    
+    long taking_micros = 0;
+    for(auto& txt_str : txt_list){
+        gettimeofday(&start, nullptr);
+        FUNASR_RESULT result=CTTransformerInfer(punc_hanlde, txt_str.c_str(), RASR_NONE, nullptr);
+        gettimeofday(&end, nullptr);
+        seconds = (end.tv_sec - start.tv_sec);
+        taking_micros += ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+        string msg = FunASRGetResult(result, 0);
+        LOG(INFO)<<"Results: "<<msg;
+        CTTransformerFreeResult(result);
+    }
+
+    LOG(INFO) << "Model inference takes: " << (double)taking_micros / 1000000 <<" s";
+    CTTransformerUninit(punc_hanlde);
+    return 0;
+}
+

runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp

@@ -0,0 +1,264 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#ifndef _WIN32
+#include <sys/time.h>
+#else
+#include <win_func.h>
+#endif
+
+#include <glog/logging.h>
+#include "funasrruntime.h"
+#include "tclap/CmdLine.h"
+#include "com-define.h"
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <vector>
+#include <atomic>
+#include <mutex>
+#include <thread>
+#include <map>
+#include <unordered_map>
+#include "util.h"
+using namespace std;
+
+std::atomic<int> wav_index(0);
+std::mutex mtx;
+
+void runReg(FUNASR_HANDLE asr_handle, vector<string> wav_list, vector<string> wav_ids, int audio_fs,
+            float* total_length, long* total_time, int core_id, float glob_beam = 3.0f, float lat_beam = 3.0f, float am_sc = 10.0f, 
+            int fst_inc_wts = 20, string hotword_path = "") {
+    
+    struct timeval start, end;
+    long seconds = 0;
+    float n_total_length = 0.0f;
+    long n_total_time = 0;
+	
+    // init wfst decoder
+    FUNASR_DEC_HANDLE decoder_handle = FunASRWfstDecoderInit(asr_handle, ASR_OFFLINE, glob_beam, lat_beam, am_sc);
+
+    // process fst hotwords list
+    unordered_map<string, int> hws_map;
+    string nn_hotwords_ = "";
+    funasr::ExtractHws(hotword_path, hws_map, nn_hotwords_);
+
+    // load hotwords list and build graph
+    FunWfstDecoderLoadHwsRes(decoder_handle, fst_inc_wts, hws_map);
+
+    std::vector<std::vector<float>> hotwords_embedding = CompileHotwordEmbedding(asr_handle, nn_hotwords_);
+    
+    // warm up
+    for (size_t i = 0; i < 1; i++)
+    {
+        FUNASR_RESULT result=FunOfflineInfer(asr_handle, wav_list[0].c_str(), RASR_NONE, nullptr, hotwords_embedding, audio_fs, true, decoder_handle);
+        if(result){
+            FunASRFreeResult(result);
+        }
+    }
+
+    while (true) {
+        // 使用原子变量获取索引并递增
+        int i = wav_index.fetch_add(1);
+        if (i >= wav_list.size()) {
+            break;
+        }
+
+        gettimeofday(&start, nullptr);
+        FUNASR_RESULT result=FunOfflineInfer(asr_handle, wav_list[i].c_str(), RASR_NONE, nullptr, hotwords_embedding, audio_fs, true, decoder_handle);
+
+        gettimeofday(&end, nullptr);
+        seconds = (end.tv_sec - start.tv_sec);
+        long taking_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+        n_total_time += taking_micros;
+
+        if(result){
+            string msg = FunASRGetResult(result, 0);
+            LOG(INFO) << "Thread: " << this_thread::get_id() << "," << wav_ids[i] << " : " << msg;
+            string stamp = FunASRGetStamp(result);
+            if(stamp !=""){
+                LOG(INFO) << "Thread: " << this_thread::get_id() << "," << wav_ids[i] << " : " << stamp;
+            }
+            string stamp_sents = FunASRGetStampSents(result);
+            if(stamp_sents !=""){
+                LOG(INFO)<< wav_ids[i] <<" : "<<stamp_sents;
+            }
+            float snippet_time = FunASRGetRetSnippetTime(result);
+            n_total_length += snippet_time;
+            FunASRFreeResult(result);
+        }else{
+            LOG(ERROR) << wav_ids[i] << (": No return data!\n");
+        }
+    }
+    {
+        lock_guard<mutex> guard(mtx);
+        *total_length += n_total_length;
+        if(*total_time < n_total_time){
+            *total_time = n_total_time;
+        }
+    }
+    FunWfstDecoderUnloadHwsRes(decoder_handle);
+    FunASRWfstDecoderUninit(decoder_handle);
+}
+
+bool is_target_file(const std::string& filename, const std::string target) {
+    std::size_t pos = filename.find_last_of(".");
+    if (pos == std::string::npos) {
+        return false;
+    }
+    std::string extension = filename.substr(pos + 1);
+    return (extension == target);
+}
+
+void GetValue(TCLAP::ValueArg<std::string>& value_arg, string key, std::map<std::string, std::string>& model_path)
+{
+    model_path.insert({key, value_arg.getValue()});
+    LOG(INFO)<< key << " : " << value_arg.getValue();
+}
+
+int main(int argc, char *argv[])
+{
+    google::InitGoogleLogging(argv[0]);
+    FLAGS_logtostderr = true;
+
+    TCLAP::CmdLine cmd("funasr-onnx-offline-rtf", ' ', "1.0");
+    TCLAP::ValueArg<std::string>    model_dir("", MODEL_DIR, "the model path, which contains model.onnx, config.yaml, am.mvn", true, "", "string");
+    TCLAP::ValueArg<std::string>    quantize("", QUANTIZE, "true (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    bladedisc("", BLADEDISC, "true (Default), load the model of bladedisc in model_dir.", false, "true", "string");
+    TCLAP::ValueArg<std::string>    vad_dir("", VAD_DIR, "the vad model path, which contains model.onnx, vad.yaml, vad.mvn", false, "", "string");
+    TCLAP::ValueArg<std::string>    vad_quant("", VAD_QUANT, "true (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    punc_dir("", PUNC_DIR, "the punc model path, which contains model.onnx, punc.yaml", false, "", "string");
+    TCLAP::ValueArg<std::string>    punc_quant("", PUNC_QUANT, "true (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    lm_dir("", LM_DIR, "the lm model path, which contains compiled models: TLG.fst, config.yaml ", false, "", "string");
+    TCLAP::ValueArg<float>    global_beam("", GLOB_BEAM, "the decoding beam for beam searching ", false, 3.0, "float");
+    TCLAP::ValueArg<float>    lattice_beam("", LAT_BEAM, "the lattice generation beam for beam searching ", false, 3.0, "float");
+    TCLAP::ValueArg<float>    am_scale("", AM_SCALE, "the acoustic scale for beam searching ", false, 10.0, "float");
+    TCLAP::ValueArg<std::int32_t>   fst_inc_wts("", FST_INC_WTS, "the fst hotwords incremental bias", false, 20, "int32_t");
+    TCLAP::ValueArg<std::string>    itn_dir("", ITN_DIR, "the itn model(fst) path, which contains zh_itn_tagger.fst and zh_itn_verbalizer.fst", false, "", "string");
+
+    TCLAP::ValueArg<std::string> wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string");
+    TCLAP::ValueArg<std::int32_t>   audio_fs("", AUDIO_FS, "the sample rate of audio", false, 16000, "int32_t");
+    TCLAP::ValueArg<std::int32_t> thread_num("", THREAD_NUM, "multi-thread num for rtf", false, 1, "int32_t");
+    TCLAP::ValueArg<std::string>    hotword("", HOTWORD, "the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)", false, "", "string");
+    TCLAP::SwitchArg use_gpu("", INFER_GPU, "Whether to use GPU for inference, default is false", false);
+    TCLAP::ValueArg<std::int32_t> batch_size("", BATCHSIZE, "batch_size for ASR model when using GPU", false, 4, "int32_t");
+
+    cmd.add(model_dir);
+    cmd.add(quantize);
+    cmd.add(bladedisc);
+    cmd.add(vad_dir);
+    cmd.add(vad_quant);
+    cmd.add(punc_dir);
+    cmd.add(punc_quant);
+    cmd.add(itn_dir);
+    cmd.add(lm_dir);
+    cmd.add(global_beam);
+    cmd.add(lattice_beam);
+    cmd.add(am_scale);
+    cmd.add(hotword);
+    cmd.add(fst_inc_wts);
+    cmd.add(wav_path);
+    cmd.add(audio_fs);
+    cmd.add(thread_num);
+    cmd.add(use_gpu);
+    cmd.add(batch_size);
+    cmd.parse(argc, argv);
+
+    std::map<std::string, std::string> model_path;
+    GetValue(model_dir, MODEL_DIR, model_path);
+    GetValue(quantize, QUANTIZE, model_path);
+    GetValue(bladedisc, BLADEDISC, model_path);
+    GetValue(vad_dir, VAD_DIR, model_path);
+    GetValue(vad_quant, VAD_QUANT, model_path);
+    GetValue(punc_dir, PUNC_DIR, model_path);
+    GetValue(punc_quant, PUNC_QUANT, model_path);
+    GetValue(itn_dir, ITN_DIR, model_path);
+    GetValue(lm_dir, LM_DIR, model_path);
+    GetValue(hotword, HOTWORD, model_path);
+    GetValue(wav_path, WAV_PATH, model_path);
+
+    struct timeval start, end;
+    gettimeofday(&start, nullptr);
+    bool use_gpu_ = use_gpu.getValue();
+    int batch_size_ = batch_size.getValue();
+    FUNASR_HANDLE asr_handle=FunOfflineInit(model_path, 1, use_gpu_, batch_size_);
+
+    if (!asr_handle)
+    {
+        LOG(ERROR) << "FunASR init failed";
+        exit(-1);
+    }
+
+    gettimeofday(&end, nullptr);
+    long seconds = (end.tv_sec - start.tv_sec);
+    long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+    LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s";
+
+    // read wav_path
+    vector<string> wav_list;
+    vector<string> wav_ids;
+    string default_id = "wav_default_id";
+    string wav_path_ = model_path.at(WAV_PATH);
+    if(is_target_file(wav_path_, "wav") || is_target_file(wav_path_, "pcm")){
+        wav_list.emplace_back(wav_path_);
+        wav_ids.emplace_back(default_id);
+    }
+    else if(is_target_file(wav_path_, "scp")){
+        ifstream in(wav_path_);
+        if (!in.is_open()) {
+            LOG(ERROR) << "Failed to open file: " << model_path.at(WAV_SCP) ;
+            return 0;
+        }
+        string line;
+        while(getline(in, line))
+        {
+            istringstream iss(line);
+            string column1, column2;
+            iss >> column1 >> column2;
+            wav_list.emplace_back(column2);
+            wav_ids.emplace_back(column1);
+        }
+        in.close();
+    }else{
+        LOG(ERROR)<<"Please check the wav extension!";
+        exit(-1);
+    }
+
+    // 多线程测试
+    float total_length = 0.0f;
+    long total_time = 0;
+    std::vector<std::thread> threads;
+
+    int rtf_threds = thread_num.getValue();
+    std::string hotword_path = hotword.getValue();
+    int value_bias = 20;
+    value_bias = fst_inc_wts.getValue();
+    float glob_beam = 3.0f;
+    float lat_beam = 3.0f;
+    float am_sc = 10.0f;
+    if (lm_dir.isSet()) {
+        glob_beam = global_beam.getValue();
+        lat_beam = lattice_beam.getValue();
+        am_sc = am_scale.getValue();
+    }
+    for (int i = 0; i < rtf_threds; i++)
+    {
+        threads.emplace_back(thread(runReg, asr_handle, wav_list, wav_ids, audio_fs.getValue(), &total_length, &total_time, i, glob_beam, lat_beam, am_sc, value_bias, hotword_path));
+    }
+
+    for (auto& thread : threads)
+    {
+        thread.join();
+    }
+
+    LOG(INFO) << "total_time_wav " << (long)(total_length * 1000) << " ms";
+    LOG(INFO) << "total_time_comput " << total_time / 1000 << " ms";
+    LOG(INFO) << "total_rtf " << (double)total_time/ (total_length*1000000);
+    LOG(INFO) << "speedup " << 1.0/((double)total_time/ (total_length*1000000));
+
+    FunOfflineUninit(asr_handle);
+    return 0;
+}

runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp

@@ -0,0 +1,160 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#ifndef _WIN32
+#include <sys/time.h>
+#else
+#include <win_func.h>
+#endif
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <map>
+#include <vector>
+#include <glog/logging.h>
+#include "funasrruntime.h"
+#include "tclap/CmdLine.h"
+#include "com-define.h"
+
+using namespace std;
+
+bool is_target_file(const std::string& filename, const std::string target) {
+    std::size_t pos = filename.find_last_of(".");
+    if (pos == std::string::npos) {
+        return false;
+    }
+    std::string extension = filename.substr(pos + 1);
+    return (extension == target);
+}
+
+void GetValue(TCLAP::ValueArg<std::string>& value_arg, string key, std::map<std::string, std::string>& model_path)
+{
+    if (value_arg.isSet()){
+        model_path.insert({key, value_arg.getValue()});
+        LOG(INFO)<< key << " : " << value_arg.getValue();
+    }
+}
+
+void print_segs(vector<vector<int>>* vec, string &wav_id) {
+    string seg_out=wav_id + ": [";
+    for (int i = 0; i < vec->size(); i++) {
+        vector<int> inner_vec = (*vec)[i];
+        seg_out += "[";
+        for (int j = 0; j < inner_vec.size(); j++) {
+            seg_out += to_string(inner_vec[j]);
+            if (j != inner_vec.size() - 1) {
+                seg_out += ",";
+            }
+        }
+        seg_out += "]";
+        if (i != vec->size() - 1) {
+            seg_out += ",";
+        }
+    }
+    seg_out += "]";
+    LOG(INFO)<<seg_out;
+}
+
+int main(int argc, char *argv[])
+{
+    google::InitGoogleLogging(argv[0]);
+    FLAGS_logtostderr = true;
+
+    TCLAP::CmdLine cmd("funasr-onnx-offline-vad", ' ', "1.0");
+    TCLAP::ValueArg<std::string>    model_dir("", MODEL_DIR, "the vad model path, which contains model.onnx, vad.yaml, vad.mvn", true, "", "string");
+    TCLAP::ValueArg<std::string>    quantize("", QUANTIZE, "true (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "true", "string");
+
+    TCLAP::ValueArg<std::string>    wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string");
+    TCLAP::ValueArg<std::int32_t>   audio_fs("", AUDIO_FS, "the sample rate of audio", false, 16000, "int32_t");
+
+    cmd.add(model_dir);
+    cmd.add(quantize);
+    cmd.add(wav_path);
+    cmd.add(audio_fs);
+    cmd.parse(argc, argv);
+
+    std::map<std::string, std::string> model_path;
+    GetValue(model_dir, MODEL_DIR, model_path);
+    GetValue(quantize, QUANTIZE, model_path);
+    GetValue(wav_path, WAV_PATH, model_path);
+
+    struct timeval start, end;
+    gettimeofday(&start, nullptr);
+    int thread_num = 1;
+    FUNASR_HANDLE vad_hanlde=FsmnVadInit(model_path, thread_num);
+
+    if (!vad_hanlde)
+    {
+        LOG(ERROR) << "FunVad init failed";
+        exit(-1);
+    }
+
+    gettimeofday(&end, nullptr);
+    long seconds = (end.tv_sec - start.tv_sec);
+    long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+    LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s";
+
+    // read wav_path
+    vector<string> wav_list;
+    vector<string> wav_ids;
+    string default_id = "wav_default_id";
+    string wav_path_ = model_path.at(WAV_PATH);
+    if(is_target_file(wav_path_, "wav") || is_target_file(wav_path_, "pcm")){
+        wav_list.emplace_back(wav_path_);
+        wav_ids.emplace_back(default_id);
+    }
+    else if(is_target_file(wav_path_, "scp")){
+        ifstream in(wav_path_);
+        if (!in.is_open()) {
+            LOG(ERROR) << "Failed to open file: " << model_path.at(WAV_SCP) ;
+            return 0;
+        }
+        string line;
+        while(getline(in, line))
+        {
+            istringstream iss(line);
+            string column1, column2;
+            iss >> column1 >> column2;
+            wav_list.emplace_back(column2);
+            wav_ids.emplace_back(column1);
+        }
+        in.close();
+    }else{
+        LOG(ERROR)<<"Please check the wav extension!";
+        exit(-1);
+    }
+    
+    float snippet_time = 0.0f;
+    long taking_micros = 0;
+    for (int i = 0; i < wav_list.size(); i++) {
+        auto& wav_file = wav_list[i];
+        auto& wav_id = wav_ids[i];
+        gettimeofday(&start, nullptr);
+        FUNASR_RESULT result=FsmnVadInfer(vad_hanlde, wav_file.c_str(), nullptr, audio_fs.getValue());
+        gettimeofday(&end, nullptr);
+        seconds = (end.tv_sec - start.tv_sec);
+        taking_micros += ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+
+        if (result)
+        {
+            vector<std::vector<int>>* vad_segments = FsmnVadGetResult(result, 0);
+            print_segs(vad_segments, wav_id);
+            snippet_time += FsmnVadGetRetSnippetTime(result);
+            FsmnVadFreeResult(result);
+        }
+        else
+        {
+            LOG(ERROR) << ("No return data!\n");
+        }
+    }
+
+    LOG(INFO) << "Audio length: " << (double)snippet_time << " s";
+    LOG(INFO) << "Model inference takes: " << (double)taking_micros / 1000000 <<" s";
+    LOG(INFO) << "Model inference RTF: " << (double)taking_micros/ (snippet_time*1000000);
+    FsmnVadUninit(vad_hanlde);
+    return 0;
+}
+

runtime/onnxruntime/bin/funasr-onnx-offline.cpp

@@ -0,0 +1,230 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#ifndef _WIN32
+#include <sys/time.h>
+#else
+#include <win_func.h>
+#endif
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <map>
+#include <glog/logging.h>
+#include "funasrruntime.h"
+#include "tclap/CmdLine.h"
+#include "com-define.h"
+#include <unordered_map>
+#include "util.h"
+#include "audio.h"
+using namespace std;
+
+bool is_target_file(const std::string& filename, const std::string target) {
+    std::size_t pos = filename.find_last_of(".");
+    if (pos == std::string::npos) {
+        return false;
+    }
+    std::string extension = filename.substr(pos + 1);
+    return (extension == target);
+}
+
+void GetValue(TCLAP::ValueArg<std::string>& value_arg, string key, std::map<std::string, std::string>& model_path)
+{
+    model_path.insert({key, value_arg.getValue()});
+    LOG(INFO)<< key << " : " << value_arg.getValue();
+}
+
+int main(int argc, char** argv)
+{
+    google::InitGoogleLogging(argv[0]);
+    FLAGS_logtostderr = true;
+
+    TCLAP::CmdLine cmd("funasr-onnx-offline", ' ', "1.0");
+    TCLAP::ValueArg<std::string>    model_dir("", MODEL_DIR, "the asr model path, which contains model.onnx, config.yaml, am.mvn", true, "", "string");
+    TCLAP::ValueArg<std::string>    quantize("", QUANTIZE, "true (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    bladedisc("", BLADEDISC, "true (Default), load the model of bladedisc in model_dir.", false, "true", "string");
+    TCLAP::ValueArg<std::string>    vad_dir("", VAD_DIR, "the vad model path, which contains model.onnx, vad.yaml, vad.mvn", false, "", "string");
+    TCLAP::ValueArg<std::string>    vad_quant("", VAD_QUANT, "true (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    punc_dir("", PUNC_DIR, "the punc model path, which contains model.onnx, punc.yaml", false, "", "string");
+    TCLAP::ValueArg<std::string>    punc_quant("", PUNC_QUANT, "true (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    lm_dir("", LM_DIR, "the lm model path, which contains compiled models: TLG.fst, config.yaml, lexicon.txt ", false, "", "string");
+    TCLAP::ValueArg<float>    global_beam("", GLOB_BEAM, "the decoding beam for beam searching ", false, 3.0, "float");
+    TCLAP::ValueArg<float>    lattice_beam("", LAT_BEAM, "the lattice generation beam for beam searching ", false, 3.0, "float");
+    TCLAP::ValueArg<float>    am_scale("", AM_SCALE, "the acoustic scale for beam searching ", false, 10.0, "float");
+    TCLAP::ValueArg<std::int32_t>   fst_inc_wts("", FST_INC_WTS, "the fst hotwords incremental bias", false, 20, "int32_t");
+    TCLAP::ValueArg<std::string>    itn_dir("", ITN_DIR, "the itn model(fst) path, which contains zh_itn_tagger.fst and zh_itn_verbalizer.fst", false, "", "string");
+    TCLAP::ValueArg<std::string>    wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string");
+    TCLAP::ValueArg<std::int32_t>   audio_fs("", AUDIO_FS, "the sample rate of audio", false, 16000, "int32_t");
+    TCLAP::ValueArg<std::string>    hotword("", HOTWORD, "the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)", false, "", "string");
+    TCLAP::SwitchArg use_gpu("", INFER_GPU, "Whether to use GPU for inference, default is false", false);
+    TCLAP::ValueArg<std::int32_t> batch_size("", BATCHSIZE, "batch_size for ASR model when using GPU", false, 4, "int32_t");
+
+    cmd.add(model_dir);
+    cmd.add(quantize);
+    cmd.add(bladedisc);
+    cmd.add(vad_dir);
+    cmd.add(vad_quant);
+    cmd.add(punc_dir);
+    cmd.add(punc_quant);
+    cmd.add(itn_dir);
+    cmd.add(lm_dir);
+    cmd.add(global_beam);
+    cmd.add(lattice_beam);
+    cmd.add(am_scale);
+    cmd.add(fst_inc_wts);
+    cmd.add(wav_path);
+    cmd.add(audio_fs);
+    cmd.add(hotword);
+    cmd.add(use_gpu);
+    cmd.add(batch_size);
+    cmd.parse(argc, argv);
+
+    std::map<std::string, std::string> model_path;
+    GetValue(model_dir, MODEL_DIR, model_path);
+    GetValue(quantize, QUANTIZE, model_path);
+    GetValue(bladedisc, BLADEDISC, model_path);
+    GetValue(vad_dir, VAD_DIR, model_path);
+    GetValue(vad_quant, VAD_QUANT, model_path);
+    GetValue(punc_dir, PUNC_DIR, model_path);
+    GetValue(punc_quant, PUNC_QUANT, model_path);
+    GetValue(itn_dir, ITN_DIR, model_path);
+    GetValue(lm_dir, LM_DIR, model_path);
+    GetValue(wav_path, WAV_PATH, model_path);
+
+    struct timeval start, end;
+    gettimeofday(&start, nullptr);
+    int thread_num = 1;
+    bool use_gpu_ = use_gpu.getValue();
+    int batch_size_ = batch_size.getValue();
+    FUNASR_HANDLE asr_hanlde=FunOfflineInit(model_path, thread_num, use_gpu_, batch_size_);
+
+    if (!asr_hanlde)
+    {
+        LOG(ERROR) << "FunASR init failed";
+        exit(-1);
+    }
+    float glob_beam = 3.0f;
+    float lat_beam = 3.0f;
+    float am_sc = 10.0f;
+    if (lm_dir.isSet()) {
+        glob_beam = global_beam.getValue();
+        lat_beam = lattice_beam.getValue();
+        am_sc = am_scale.getValue();
+    }
+    // init wfst decoder
+    FUNASR_DEC_HANDLE decoder_handle = FunASRWfstDecoderInit(asr_hanlde, ASR_OFFLINE, glob_beam, lat_beam, am_sc);
+
+    // hotword file
+    unordered_map<string, int> hws_map;
+    std::string nn_hotwords_ = "";
+    std::string hotword_path = hotword.getValue();
+    LOG(INFO) << "hotword path: " << hotword_path;
+    funasr::ExtractHws(hotword_path, hws_map, nn_hotwords_);
+
+    gettimeofday(&end, nullptr);
+    long seconds = (end.tv_sec - start.tv_sec);
+    long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+    LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s";
+
+    // read wav_path
+    vector<string> wav_list;
+    vector<string> wav_ids;
+    string default_id = "wav_default_id";
+    string wav_path_ = model_path.at(WAV_PATH);
+
+    if(is_target_file(wav_path_, "scp")){
+        ifstream in(wav_path_);
+        if (!in.is_open()) {
+            LOG(ERROR) << "Failed to open file: " << model_path.at(WAV_SCP) ;
+            return 0;
+        }
+        string line;
+        while(getline(in, line))
+        {
+            istringstream iss(line);
+            string column1, column2;
+            iss >> column1 >> column2;
+            wav_list.emplace_back(column2);
+            wav_ids.emplace_back(column1);
+        }
+        in.close();
+    }else{
+        wav_list.emplace_back(wav_path_);
+        wav_ids.emplace_back(default_id);
+    }
+    
+    float snippet_time = 0.0f;
+    long taking_micros = 0;
+
+    // load hotwords list and build graph
+    FunWfstDecoderLoadHwsRes(decoder_handle, fst_inc_wts.getValue(), hws_map);
+	
+    std::vector<std::vector<float>> hotwords_embedding = CompileHotwordEmbedding(asr_hanlde, nn_hotwords_);
+    for (int i = 0; i < wav_list.size(); i++) {
+        auto& wav_file = wav_list[i];
+        auto& wav_id = wav_ids[i];
+
+        // For debug:begin
+        // int32_t sampling_rate_ = audio_fs.getValue();
+        // funasr::Audio audio(1);
+		// if(is_target_file(wav_file.c_str(), "wav")){
+		// 	if(!audio.LoadWav2Char(wav_file.c_str(), &sampling_rate_)){
+		// 		LOG(ERROR)<<"Failed to load "<< wav_file;
+        //         exit(-1);
+        //     }
+		// }else if(is_target_file(wav_file.c_str(), "pcm")){
+		// 	if (!audio.LoadPcmwav2Char(wav_file.c_str(), &sampling_rate_)){
+		// 		LOG(ERROR)<<"Failed to load "<< wav_file;
+        //         exit(-1);
+        //     }
+		// }else{
+		// 	if (!audio.FfmpegLoad(wav_file.c_str(), true)){
+		// 		LOG(ERROR)<<"Failed to load "<< wav_file;
+        //         exit(-1);
+        //     }
+		// }
+        // char* speech_buff = audio.GetSpeechChar();
+        // int buff_len = audio.GetSpeechLen()*2;
+
+        // gettimeofday(&start, nullptr);
+        // FUNASR_RESULT result=FunOfflineInferBuffer(asr_hanlde, speech_buff, buff_len, RASR_NONE, nullptr, hotwords_embedding, audio_fs.getValue(), "pcm", true, decoder_handle);
+        // For debug:end
+
+        FUNASR_RESULT result=FunOfflineInfer(asr_hanlde, wav_file.c_str(), RASR_NONE, nullptr, hotwords_embedding, audio_fs.getValue(), true, decoder_handle);
+        gettimeofday(&end, nullptr);
+        seconds = (end.tv_sec - start.tv_sec);
+        taking_micros += ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+
+        if (result)
+        {
+            string msg = FunASRGetResult(result, 0);
+            LOG(INFO)<< wav_id <<" : "<<msg;
+            string stamp = FunASRGetStamp(result);
+            if(stamp !=""){
+                LOG(INFO)<< wav_id <<" : "<<stamp;
+            }
+            string stamp_sents = FunASRGetStampSents(result);
+            if(stamp_sents !=""){
+                LOG(INFO)<< wav_id <<" : "<<stamp_sents;
+            }
+            snippet_time += FunASRGetRetSnippetTime(result);
+            FunASRFreeResult(result);
+        }
+        else
+        {
+            LOG(ERROR) << ("No return data!\n");
+        }
+    }
+    FunWfstDecoderUnloadHwsRes(decoder_handle);
+    LOG(INFO) << "Audio length: " << (double)snippet_time << " s";
+    LOG(INFO) << "Model inference takes: " << (double)taking_micros / 1000000 <<" s";
+    LOG(INFO) << "Model inference RTF: " << (double)taking_micros/ (snippet_time*1000000);
+    
+    FunASRWfstDecoderUninit(decoder_handle);
+    FunOfflineUninit(asr_hanlde);
+    return 0;
+}
+

runtime/onnxruntime/bin/funasr-onnx-online-asr.cpp

@@ -0,0 +1,176 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#ifndef _WIN32
+#include <sys/time.h>
+#else
+#include <win_func.h>
+#endif
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <map>
+#include <vector>
+#include <glog/logging.h>
+#include "funasrruntime.h"
+#include "tclap/CmdLine.h"
+#include "com-define.h"
+#include "audio.h"
+
+using namespace std;
+
+bool is_target_file(const std::string& filename, const std::string target) {
+    std::size_t pos = filename.find_last_of(".");
+    if (pos == std::string::npos) {
+        return false;
+    }
+    std::string extension = filename.substr(pos + 1);
+    return (extension == target);
+}
+
+void GetValue(TCLAP::ValueArg<std::string>& value_arg, string key, std::map<std::string, std::string>& model_path)
+{
+    if (value_arg.isSet()){
+        model_path.insert({key, value_arg.getValue()});
+        LOG(INFO)<< key << " : " << value_arg.getValue();
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    google::InitGoogleLogging(argv[0]);
+    FLAGS_logtostderr = true;
+
+    TCLAP::CmdLine cmd("funasr-onnx-offline-vad", ' ', "1.0");
+    TCLAP::ValueArg<std::string>    model_dir("", MODEL_DIR, "the asr online model path, which contains model.onnx, decoder.onnx, config.yaml, am.mvn", true, "", "string");
+    TCLAP::ValueArg<std::string>    quantize("", QUANTIZE, "true (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "true", "string");
+
+    TCLAP::ValueArg<std::string>    wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string");
+    TCLAP::ValueArg<std::int32_t>   audio_fs("", AUDIO_FS, "the sample rate of audio", false, 16000, "int32_t");
+
+    cmd.add(model_dir);
+    cmd.add(quantize);
+    cmd.add(wav_path);
+    cmd.add(audio_fs);
+    cmd.parse(argc, argv);
+
+    std::map<std::string, std::string> model_path;
+    GetValue(model_dir, MODEL_DIR, model_path);
+    GetValue(quantize, QUANTIZE, model_path);
+    GetValue(wav_path, WAV_PATH, model_path);
+
+    struct timeval start, end;
+    gettimeofday(&start, nullptr);
+    int thread_num = 1;
+    FUNASR_HANDLE asr_handle=FunASRInit(model_path, thread_num, ASR_ONLINE);
+
+    if (!asr_handle)
+    {
+        LOG(ERROR) << "FunVad init failed";
+        exit(-1);
+    }
+
+    gettimeofday(&end, nullptr);
+    long seconds = (end.tv_sec - start.tv_sec);
+    long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+    LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s";
+
+    // read wav_path
+    vector<string> wav_list;
+    vector<string> wav_ids;
+    string default_id = "wav_default_id";
+    string wav_path_ = model_path.at(WAV_PATH);
+    if(is_target_file(wav_path_, "scp")){
+        ifstream in(wav_path_);
+        if (!in.is_open()) {
+            LOG(ERROR) << "Failed to open file: " << model_path.at(WAV_SCP) ;
+            return 0;
+        }
+        string line;
+        while(getline(in, line))
+        {
+            istringstream iss(line);
+            string column1, column2;
+            iss >> column1 >> column2;
+            wav_list.emplace_back(column2);
+            wav_ids.emplace_back(column1);
+        }
+        in.close();
+    }else{
+        wav_list.emplace_back(wav_path_);
+        wav_ids.emplace_back(default_id);
+    }
+
+    // init online features
+    FUNASR_HANDLE online_handle=FunASROnlineInit(asr_handle);
+    float snippet_time = 0.0f;
+    long taking_micros = 0;
+    for (int i = 0; i < wav_list.size(); i++) {
+        auto& wav_file = wav_list[i];
+        auto& wav_id = wav_ids[i];
+
+        int32_t sampling_rate_ = audio_fs.getValue();
+        funasr::Audio audio(1);
+		if(is_target_file(wav_file.c_str(), "wav")){
+			if(!audio.LoadWav2Char(wav_file.c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_file;
+                exit(-1);
+            }
+		}else if(is_target_file(wav_file.c_str(), "pcm")){
+			if (!audio.LoadPcmwav2Char(wav_file.c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_file;
+                exit(-1);
+            }
+		}else{
+			if (!audio.FfmpegLoad(wav_file.c_str(), true)){
+				LOG(ERROR)<<"Failed to load "<< wav_file;
+                exit(-1);
+            }
+		}
+        char* speech_buff = audio.GetSpeechChar();
+        int buff_len = audio.GetSpeechLen()*2;
+
+        int step = 9600*2;
+        bool is_final = false;
+
+        string final_res="";
+        for (int sample_offset = 0; sample_offset < buff_len; sample_offset += std::min(step, buff_len - sample_offset)) {
+            if (sample_offset + step >= buff_len - 1) {
+                    step = buff_len - sample_offset;
+                    is_final = true;
+                } else {
+                    is_final = false;
+            }
+            gettimeofday(&start, nullptr);
+            FUNASR_RESULT result = FunASRInferBuffer(online_handle, speech_buff+sample_offset, step, RASR_NONE, nullptr, is_final, sampling_rate_);
+            gettimeofday(&end, nullptr);
+            seconds = (end.tv_sec - start.tv_sec);
+            taking_micros += ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+
+            if (result)
+            {
+                string msg = FunASRGetResult(result, 0);
+                final_res += msg;
+                LOG(INFO)<< wav_id <<" : "<<msg;
+                snippet_time += FunASRGetRetSnippetTime(result);
+                FunASRFreeResult(result);
+            }
+            else
+            {
+                LOG(ERROR) << ("No return data!\n");
+            }
+        }
+        LOG(INFO)<<"Final results " << wav_id <<" : "<<final_res;
+    }
+
+    LOG(INFO) << "Audio length: " << (double)snippet_time << " s";
+    LOG(INFO) << "Model inference takes: " << (double)taking_micros / 1000000 <<" s";
+    LOG(INFO) << "Model inference RTF: " << (double)taking_micros/ (snippet_time*1000000);
+    FunASRUninit(asr_handle);
+    FunASRUninit(online_handle);
+    return 0;
+}
+

runtime/onnxruntime/bin/funasr-onnx-online-punc.cpp

@@ -0,0 +1,130 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#ifndef _WIN32
+#include <sys/time.h>
+#else
+#include <win_func.h>
+#endif
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <map>
+#include <glog/logging.h>
+#include "funasrruntime.h"
+#include "tclap/CmdLine.h"
+#include "com-define.h"
+
+using namespace std;
+
+void GetValue(TCLAP::ValueArg<std::string>& value_arg, string key, std::map<std::string, std::string>& model_path)
+{
+    if (value_arg.isSet()){
+        model_path.insert({key, value_arg.getValue()});
+        LOG(INFO)<< key << " : " << value_arg.getValue();
+    }
+}
+
+void splitString(vector<string>& strings, const string& org_string, const string& seq) {
+	string::size_type p1 = 0;
+	string::size_type p2 = org_string.find(seq);
+
+	while (p2 != string::npos) {
+		if (p2 == p1) {
+			++p1;
+			p2 = org_string.find(seq, p1);
+			continue;
+		}
+		strings.push_back(org_string.substr(p1, p2 - p1));
+		p1 = p2 + seq.size();
+		p2 = org_string.find(seq, p1);
+	}
+
+	if (p1 != org_string.size()) {
+		strings.push_back(org_string.substr(p1));
+	}
+}
+
+int main(int argc, char *argv[])
+{
+    google::InitGoogleLogging(argv[0]);
+    FLAGS_logtostderr = true;
+
+    TCLAP::CmdLine cmd("funasr-onnx-online-punc", ' ', "1.0");
+    TCLAP::ValueArg<std::string>    model_dir("", MODEL_DIR, "the punc model path, which contains model.onnx, punc.yaml", true, "", "string");
+    TCLAP::ValueArg<std::string>    quantize("", QUANTIZE, "true (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string> txt_path("", TXT_PATH, "txt file path, one sentence per line", true, "", "string");
+
+    cmd.add(model_dir);
+    cmd.add(quantize);
+    cmd.add(txt_path);
+    cmd.parse(argc, argv);
+
+    std::map<std::string, std::string> model_path;
+    GetValue(model_dir, MODEL_DIR, model_path);
+    GetValue(quantize, QUANTIZE, model_path);
+    GetValue(txt_path, TXT_PATH, model_path);
+
+    struct timeval start, end;
+    gettimeofday(&start, nullptr);
+    int thread_num = 1;
+    FUNASR_HANDLE punc_hanlde=CTTransformerInit(model_path, thread_num, PUNC_ONLINE);
+
+    if (!punc_hanlde)
+    {
+        LOG(ERROR) << "FunASR init failed";
+        exit(-1);
+    }
+
+    gettimeofday(&end, nullptr);
+    long seconds = (end.tv_sec - start.tv_sec);
+    long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+    LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s";
+
+    // read txt_path
+    vector<string> txt_list;
+
+    if(model_path.find(TXT_PATH)!=model_path.end()){
+        ifstream in(model_path.at(TXT_PATH));
+        if (!in.is_open()) {
+            LOG(ERROR) << "Failed to open file: " << model_path.at(TXT_PATH) ;
+            return 0;
+        }
+        string line;
+        while(getline(in, line))
+        {
+            txt_list.emplace_back(line); 
+        }
+        in.close();
+    }
+    
+    long taking_micros = 0;
+    for(auto& txt_str : txt_list){
+        vector<string> vad_strs;
+        splitString(vad_strs, txt_str, "|");
+        string str_out;
+        FUNASR_RESULT result = nullptr;
+        gettimeofday(&start, nullptr);
+        for(auto& vad_str:vad_strs){
+            result=CTTransformerInfer(punc_hanlde, vad_str.c_str(), RASR_NONE, nullptr, PUNC_ONLINE, result);
+            if(result){
+                string msg = CTTransformerGetResult(result, 0);
+                str_out += msg;
+                LOG(INFO)<<"Online result: "<<msg;
+            }
+        }
+        gettimeofday(&end, nullptr);
+        seconds = (end.tv_sec - start.tv_sec);
+        taking_micros += ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+        LOG(INFO)<<"Results: "<<str_out;
+        CTTransformerFreeResult(result);
+    }
+
+    LOG(INFO) << "Model inference takes: " << (double)taking_micros / 1000000 <<" s";
+    CTTransformerUninit(punc_hanlde);
+    return 0;
+}
+

runtime/onnxruntime/bin/funasr-onnx-online-rtf.cpp

@@ -0,0 +1,280 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#ifndef _WIN32
+#include <sys/time.h>
+#else
+#include <win_func.h>
+#endif
+
+#include <glog/logging.h>
+#include "funasrruntime.h"
+#include "tclap/CmdLine.h"
+#include "com-define.h"
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <vector>
+#include <atomic>
+#include <mutex>
+#include <thread>
+#include <map>
+#include "audio.h"
+
+using namespace std;
+
+std::atomic<int> wav_index(0);
+std::mutex mtx;
+
+bool is_target_file(const std::string& filename, const std::string target) {
+    std::size_t pos = filename.find_last_of(".");
+    if (pos == std::string::npos) {
+        return false;
+    }
+    std::string extension = filename.substr(pos + 1);
+    return (extension == target);
+}
+
+void runReg(FUNASR_HANDLE asr_handle, vector<string> wav_list, vector<string> wav_ids, int audio_fs,
+            float* total_length, long* total_time, int core_id) {
+    
+    struct timeval start, end;
+    long seconds = 0;
+    float n_total_length = 0.0f;
+    long n_total_time = 0;
+    
+    // init online features
+    FUNASR_HANDLE online_handle=FunASROnlineInit(asr_handle);
+
+    // warm up
+    for (size_t i = 0; i < 10; i++)
+    {
+        int32_t sampling_rate_ = audio_fs;
+        funasr::Audio audio(1);
+		if(is_target_file(wav_list[0].c_str(), "wav")){
+			if(!audio.LoadWav2Char(wav_list[0].c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_list[0];
+                exit(-1);
+            }
+		}else if(is_target_file(wav_list[0].c_str(), "pcm")){
+			if (!audio.LoadPcmwav2Char(wav_list[0].c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_list[0];
+                exit(-1);
+            }
+		}else{
+			if (!audio.FfmpegLoad(wav_list[0].c_str(), true)){
+				LOG(ERROR)<<"Failed to load "<< wav_list[0];
+                exit(-1);
+            }
+		}
+        char* speech_buff = audio.GetSpeechChar();
+        int buff_len = audio.GetSpeechLen()*2;
+
+        int step = 9600*2;
+        bool is_final = false;
+
+        string final_res="";
+        for (int sample_offset = 0; sample_offset < buff_len; sample_offset += std::min(step, buff_len - sample_offset)) {
+            if (sample_offset + step >= buff_len - 1) {
+                    step = buff_len - sample_offset;
+                    is_final = true;
+                } else {
+                    is_final = false;
+            }
+            FUNASR_RESULT result = FunASRInferBuffer(online_handle, speech_buff+sample_offset, step, RASR_NONE, nullptr, is_final, sampling_rate_);
+            if (result)
+            {
+                FunASRFreeResult(result);
+            }
+        }
+    }
+
+    while (true) {
+        // 使用原子变量获取索引并递增
+        int i = wav_index.fetch_add(1);
+        if (i >= wav_list.size()) {
+            break;
+        }
+        int32_t sampling_rate_ = audio_fs;
+        funasr::Audio audio(1);
+		if(is_target_file(wav_list[i].c_str(), "wav")){
+			if(!audio.LoadWav2Char(wav_list[i].c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_list[i];
+                exit(-1);
+            }
+		}else if(is_target_file(wav_list[i].c_str(), "pcm")){
+			if (!audio.LoadPcmwav2Char(wav_list[i].c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_list[i];
+                exit(-1);
+            }
+		}else{
+			if (!audio.FfmpegLoad(wav_list[i].c_str(), true)){
+				LOG(ERROR)<<"Failed to load "<< wav_list[i];
+                exit(-1);
+            }
+		}
+        char* speech_buff = audio.GetSpeechChar();
+        int buff_len = audio.GetSpeechLen()*2;
+
+        int step = 9600*2;
+        bool is_final = false;
+
+        string final_res="";
+        for (int sample_offset = 0; sample_offset < buff_len; sample_offset += std::min(step, buff_len - sample_offset)) {
+            if (sample_offset + step >= buff_len - 1) {
+                    step = buff_len - sample_offset;
+                    is_final = true;
+                } else {
+                    is_final = false;
+            }
+            gettimeofday(&start, nullptr);
+            FUNASR_RESULT result = FunASRInferBuffer(online_handle, speech_buff+sample_offset, step, RASR_NONE, nullptr, is_final, sampling_rate_);
+            gettimeofday(&end, nullptr);
+            seconds = (end.tv_sec - start.tv_sec);
+            long taking_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+            n_total_time += taking_micros;
+
+            if (result)
+            {
+                string msg = FunASRGetResult(result, 0);
+                final_res += msg;
+                LOG(INFO) << "Thread: " << this_thread::get_id() << "," << wav_ids[i] << " : " << msg;
+                float snippet_time = FunASRGetRetSnippetTime(result);
+                n_total_length += snippet_time;
+                FunASRFreeResult(result);
+            }
+            else
+            {
+                LOG(ERROR) << ("No return data!\n");
+            }
+        }
+        LOG(INFO) << "Thread: " << this_thread::get_id() << ", Final results " << wav_ids[i] << " : " << final_res;
+
+    }
+    {
+        lock_guard<mutex> guard(mtx);
+        *total_length += n_total_length;
+        if(*total_time < n_total_time){
+            *total_time = n_total_time;
+        }
+    }
+    FunASRUninit(online_handle);
+}
+
+void GetValue(TCLAP::ValueArg<std::string>& value_arg, string key, std::map<std::string, std::string>& model_path)
+{
+    if (value_arg.isSet()){
+        model_path.insert({key, value_arg.getValue()});
+        LOG(INFO)<< key << " : " << value_arg.getValue();
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    google::InitGoogleLogging(argv[0]);
+    FLAGS_logtostderr = true;
+
+    TCLAP::CmdLine cmd("funasr-onnx-online-rtf", ' ', "1.0");
+    TCLAP::ValueArg<std::string>    model_dir("", MODEL_DIR, "the model path, which contains model.onnx, config.yaml, am.mvn", true, "", "string");
+    TCLAP::ValueArg<std::string>    quantize("", QUANTIZE, "true (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    vad_dir("", VAD_DIR, "the vad model path, which contains model.onnx, vad.yaml, vad.mvn", false, "", "string");
+    TCLAP::ValueArg<std::string>    vad_quant("", VAD_QUANT, "true (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir", false, "true", "string");
+    TCLAP::ValueArg<std::string>    punc_dir("", PUNC_DIR, "the punc model path, which contains model.onnx, punc.yaml", false, "", "string");
+    TCLAP::ValueArg<std::string>    punc_quant("", PUNC_QUANT, "true (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir", false, "true", "string");
+
+    TCLAP::ValueArg<std::string> wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string");
+    TCLAP::ValueArg<std::int32_t>   audio_fs("", AUDIO_FS, "the sample rate of audio", false, 16000, "int32_t");
+    TCLAP::ValueArg<std::int32_t> thread_num("", THREAD_NUM, "multi-thread num for rtf", true, 0, "int32_t");
+
+    cmd.add(model_dir);
+    cmd.add(quantize);
+    cmd.add(vad_dir);
+    cmd.add(vad_quant);
+    cmd.add(punc_dir);
+    cmd.add(punc_quant);
+    cmd.add(wav_path);
+    cmd.add(audio_fs);
+    cmd.add(thread_num);
+    cmd.parse(argc, argv);
+
+    std::map<std::string, std::string> model_path;
+    GetValue(model_dir, MODEL_DIR, model_path);
+    GetValue(quantize, QUANTIZE, model_path);
+    GetValue(vad_dir, VAD_DIR, model_path);
+    GetValue(vad_quant, VAD_QUANT, model_path);
+    GetValue(punc_dir, PUNC_DIR, model_path);
+    GetValue(punc_quant, PUNC_QUANT, model_path);
+    GetValue(wav_path, WAV_PATH, model_path);
+
+    struct timeval start, end;
+    gettimeofday(&start, nullptr);
+    FUNASR_HANDLE asr_handle=FunASRInit(model_path, 1, ASR_ONLINE);
+
+    if (!asr_handle)
+    {
+        LOG(ERROR) << "FunASR init failed";
+        exit(-1);
+    }
+
+    gettimeofday(&end, nullptr);
+    long seconds = (end.tv_sec - start.tv_sec);
+    long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+    LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s";
+
+    // read wav_path
+    vector<string> wav_list;
+    vector<string> wav_ids;
+    string default_id = "wav_default_id";
+    string wav_path_ = model_path.at(WAV_PATH);
+    if(is_target_file(wav_path_, "wav") || is_target_file(wav_path_, "pcm")){
+        wav_list.emplace_back(wav_path_);
+        wav_ids.emplace_back(default_id);
+    }
+    else if(is_target_file(wav_path_, "scp")){
+        ifstream in(wav_path_);
+        if (!in.is_open()) {
+            LOG(ERROR) << "Failed to open file: " << model_path.at(WAV_SCP) ;
+            return 0;
+        }
+        string line;
+        while(getline(in, line))
+        {
+            istringstream iss(line);
+            string column1, column2;
+            iss >> column1 >> column2;
+            wav_list.emplace_back(column2);
+            wav_ids.emplace_back(column1);
+        }
+        in.close();
+    }else{
+        LOG(ERROR)<<"Please check the wav extension!";
+        exit(-1);
+    }
+
+    // 多线程测试
+    float total_length = 0.0f;
+    long total_time = 0;
+    std::vector<std::thread> threads;
+
+    int rtf_threds = thread_num.getValue();
+    for (int i = 0; i < rtf_threds; i++)
+    {
+        threads.emplace_back(thread(runReg, asr_handle, wav_list, wav_ids, audio_fs.getValue(), &total_length, &total_time, i));
+    }
+
+    for (auto& thread : threads)
+    {
+        thread.join();
+    }
+
+    LOG(INFO) << "total_time_wav " << (long)(total_length * 1000) << " ms";
+    LOG(INFO) << "total_time_comput " << total_time / 1000 << " ms";
+    LOG(INFO) << "total_rtf " << (double)total_time/ (total_length*1000000);
+    LOG(INFO) << "speedup " << 1.0/((double)total_time/ (total_length*1000000));
+
+    FunASRUninit(asr_handle);
+    return 0;
+}

runtime/onnxruntime/bin/funasr-onnx-online-vad.cpp

@@ -0,0 +1,200 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#ifndef _WIN32
+#include <sys/time.h>
+#else
+#include <win_func.h>
+#endif
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <map>
+#include <vector>
+#include <glog/logging.h>
+#include "funasrruntime.h"
+#include "tclap/CmdLine.h"
+#include "com-define.h"
+#include "audio.h"
+
+using namespace std;
+
+bool is_target_file(const std::string& filename, const std::string target) {
+    std::size_t pos = filename.find_last_of(".");
+    if (pos == std::string::npos) {
+        return false;
+    }
+    std::string extension = filename.substr(pos + 1);
+    return (extension == target);
+}
+
+void GetValue(TCLAP::ValueArg<std::string>& value_arg, string key, std::map<std::string, std::string>& model_path)
+{
+    if (value_arg.isSet()){
+        model_path.insert({key, value_arg.getValue()});
+        LOG(INFO)<< key << " : " << value_arg.getValue();
+    }
+}
+
+void print_segs(vector<vector<int>>* vec, string &wav_id) {
+    if((*vec).size() == 0){
+        return;
+    }    
+    string seg_out=wav_id + ": [";
+    for (int i = 0; i < vec->size(); i++) {
+        vector<int> inner_vec = (*vec)[i];
+        if(inner_vec.size() == 0){
+            continue;
+        }
+        seg_out += "[";
+        for (int j = 0; j < inner_vec.size(); j++) {
+            seg_out += to_string(inner_vec[j]);
+            if (j != inner_vec.size() - 1) {
+                seg_out += ",";
+            }
+        }
+        seg_out += "]";
+        if (i != vec->size() - 1) {
+            seg_out += ",";
+        }
+    }
+    seg_out += "]";
+    LOG(INFO)<<seg_out;
+}
+
+int main(int argc, char *argv[])
+{
+    google::InitGoogleLogging(argv[0]);
+    FLAGS_logtostderr = true;
+
+    TCLAP::CmdLine cmd("funasr-onnx-offline-vad", ' ', "1.0");
+    TCLAP::ValueArg<std::string>    model_dir("", MODEL_DIR, "the vad model path, which contains model.onnx, vad.yaml, vad.mvn", true, "", "string");
+    TCLAP::ValueArg<std::string>    quantize("", QUANTIZE, "false (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "true", "string");
+
+    TCLAP::ValueArg<std::string>    wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string");
+    TCLAP::ValueArg<std::int32_t>   audio_fs("", AUDIO_FS, "the sample rate of audio", false, 16000, "int32_t");
+
+    cmd.add(model_dir);
+    cmd.add(quantize);
+    cmd.add(wav_path);
+    cmd.add(audio_fs);
+    cmd.parse(argc, argv);
+
+    std::map<std::string, std::string> model_path;
+    GetValue(model_dir, MODEL_DIR, model_path);
+    GetValue(quantize, QUANTIZE, model_path);
+    GetValue(wav_path, WAV_PATH, model_path);
+
+    struct timeval start, end;
+    gettimeofday(&start, nullptr);
+    int thread_num = 1;
+    FUNASR_HANDLE vad_hanlde=FsmnVadInit(model_path, thread_num);
+
+    if (!vad_hanlde)
+    {
+        LOG(ERROR) << "FunVad init failed";
+        exit(-1);
+    }
+
+    gettimeofday(&end, nullptr);
+    long seconds = (end.tv_sec - start.tv_sec);
+    long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+    LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s";
+
+    // read wav_path
+    vector<string> wav_list;
+    vector<string> wav_ids;
+    string default_id = "wav_default_id";
+    string wav_path_ = model_path.at(WAV_PATH);
+    if(is_target_file(wav_path_, "wav") || is_target_file(wav_path_, "pcm")){
+        wav_list.emplace_back(wav_path_);
+        wav_ids.emplace_back(default_id);
+    }
+    else if(is_target_file(wav_path_, "scp")){
+        ifstream in(wav_path_);
+        if (!in.is_open()) {
+            LOG(ERROR) << "Failed to open file: " << model_path.at(WAV_SCP) ;
+            return 0;
+        }
+        string line;
+        while(getline(in, line))
+        {
+            istringstream iss(line);
+            string column1, column2;
+            iss >> column1 >> column2;
+            wav_list.emplace_back(column2);
+            wav_ids.emplace_back(column1);
+        }
+        in.close();
+    }else{
+        LOG(ERROR)<<"Please check the wav extension!";
+        exit(-1);
+    }
+    // init online features
+    FUNASR_HANDLE online_hanlde=FsmnVadOnlineInit(vad_hanlde);
+    float snippet_time = 0.0f;
+    long taking_micros = 0;
+    for (int i = 0; i < wav_list.size(); i++) {
+        auto& wav_file = wav_list[i];
+        auto& wav_id = wav_ids[i];
+
+        int32_t sampling_rate_ = audio_fs.getValue();
+        funasr::Audio audio(1);
+		if(is_target_file(wav_file.c_str(), "wav")){
+			if(!audio.LoadWav2Char(wav_file.c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_file;
+                exit(-1);
+            }
+		}else if(is_target_file(wav_file.c_str(), "pcm")){
+			if (!audio.LoadPcmwav2Char(wav_file.c_str(), &sampling_rate_)){
+				LOG(ERROR)<<"Failed to load "<< wav_file;
+                exit(-1);
+            }
+		}else{
+			LOG(ERROR)<<"Wrong wav extension";
+			exit(-1);
+		}
+        char* speech_buff = audio.GetSpeechChar();
+        int buff_len = audio.GetSpeechLen()*2;
+
+        int step = 800*2;
+        bool is_final = false;
+
+        for (int sample_offset = 0; sample_offset < buff_len; sample_offset += std::min(step, buff_len - sample_offset)) {
+            if (sample_offset + step >= buff_len - 1) {
+                    step = buff_len - sample_offset;
+                    is_final = true;
+                } else {
+                    is_final = false;
+            }
+            gettimeofday(&start, nullptr);
+            FUNASR_RESULT result = FsmnVadInferBuffer(online_hanlde, speech_buff+sample_offset, step, nullptr, is_final, sampling_rate_);
+            gettimeofday(&end, nullptr);
+            seconds = (end.tv_sec - start.tv_sec);
+            taking_micros += ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+
+            if (result)
+            {
+                vector<std::vector<int>>* vad_segments = FsmnVadGetResult(result, 0);
+                print_segs(vad_segments, wav_id);
+                snippet_time += FsmnVadGetRetSnippetTime(result);
+                FsmnVadFreeResult(result);
+            }
+            else
+            {
+                LOG(ERROR) << ("No return data!\n");
+            }
+        }
+    }
+
+    LOG(INFO) << "Audio length: " << (double)snippet_time << " s";
+    LOG(INFO) << "Model inference takes: " << (double)taking_micros / 1000000 <<" s";
+    LOG(INFO) << "Model inference RTF: " << (double)taking_micros/ (snippet_time*1000000);
+    FsmnVadUninit(online_hanlde);
+    FsmnVadUninit(vad_hanlde);
+    return 0;
+}
+

runtime/onnxruntime/include/audio.h

@@ -0,0 +1,118 @@
+#ifndef AUDIO_H
+#define AUDIO_H
+
+#include <queue>
+#include <stdint.h>
+#include "vad-model.h"
+#include "offline-stream.h"
+#include "com-define.h"
+
+#ifndef WAV_HEADER_SIZE
+#define WAV_HEADER_SIZE 44
+#endif
+
+using namespace std;
+namespace funasr {
+
+class AudioFrame {
+  private:
+    int start;
+    int end;
+
+
+  public:
+    AudioFrame();
+    AudioFrame(int len);
+    AudioFrame(const AudioFrame &other);
+    AudioFrame(int start, int end, bool is_final);
+
+    ~AudioFrame();
+    int SetStart(int val);
+    int SetEnd(int val);
+    int GetStart();
+    int GetLen();
+    int Disp();
+    // 2pass
+    bool is_final = false;
+    float* data = nullptr;
+    int len;
+    int global_start = 0; // the start of a frame in the global time axis. in ms
+    int global_end = 0;   // the end of a frame in the global time axis. in ms
+};
+
+#ifdef _WIN32
+#ifdef _FUNASR_API_EXPORT
+#define DLLAPI __declspec(dllexport)
+#else
+#define DLLAPI __declspec(dllimport)
+#endif
+#else
+#define DLLAPI 
+#endif
+class DLLAPI Audio {
+  private:
+    float *speech_data=nullptr;
+    int16_t *speech_buff=nullptr;
+    char* speech_char=nullptr;
+    int speech_len;
+    int speech_align_len;
+    float align_size;
+    int data_type;
+    queue<AudioFrame *> frame_queue;
+    queue<AudioFrame *> asr_online_queue;
+    queue<AudioFrame *> asr_offline_queue;
+    int dest_sample_rate;
+  public:
+    Audio(int data_type);
+    Audio(int model_sample_rate,int data_type);
+    Audio(int model_sample_rate,int data_type, int size);
+    ~Audio();
+    void ClearQueue(std::queue<AudioFrame*>& q);
+    void Disp();
+    void WavResample(int32_t sampling_rate, const float *waveform, int32_t n);
+    bool LoadWav(const char* buf, int n_len, int32_t* sampling_rate);
+    bool LoadWav(const char* filename, int32_t* sampling_rate, bool resample=true);
+    bool LoadWav2Char(const char* filename, int32_t* sampling_rate);
+    bool LoadPcmwav(const char* buf, int n_file_len, int32_t* sampling_rate);
+    bool LoadPcmwav(const char* filename, int32_t* sampling_rate, bool resample=true);
+    bool LoadPcmwav2Char(const char* filename, int32_t* sampling_rate);
+    bool LoadOthers2Char(const char* filename);
+    bool FfmpegLoad(const char *filename, bool copy2char=false);
+    bool FfmpegLoad(const char* buf, int n_file_len);
+    int FetchChunck(AudioFrame *&frame);
+    int FetchTpass(AudioFrame *&frame);
+    int Fetch(float *&dout, int &len, int &flag);
+    int Fetch(float *&dout, int &len, int &flag, float &start_time);
+    int Fetch(float **&dout, int *&len, int *&flag, float*& start_time, int batch_size, int &batch_in);
+    int FetchDynamic(float **&dout, int *&len, int *&flag, float*& start_time, int batch_size, int &batch_in);
+    void Padding();
+    void Split(OfflineStream* offline_streamj);
+    void CutSplit(OfflineStream* offline_streamj, std::vector<int> &index_vector);
+    void Split(VadModel* vad_obj, vector<std::vector<int>>& vad_segments, bool input_finished=true);
+    void Split(VadModel* vad_obj, int chunk_len, bool input_finished=true, ASR_TYPE asr_mode=ASR_TWO_PASS);
+    float GetTimeLen();
+    int GetQueueSize() { return (int)frame_queue.size(); }
+    char* GetSpeechChar(){return speech_char;}
+    int GetSpeechLen(){return speech_len;}
+
+    // 2pass
+    vector<float> all_samples;
+    int offset = 0;
+    int speech_start=-1, speech_end=0;
+    int speech_offline_start=-1;
+    int64_t start = 0;
+    int64_t end = 0;
+
+    int seg_sample = MODEL_SAMPLE_RATE/1000;
+    bool LoadPcmwavOnline(const char* buf, int n_file_len, int32_t* sampling_rate);
+    void ResetIndex(){
+      speech_start=-1;
+      speech_end=0;
+      speech_offline_start=-1;
+      offset = 0;
+      all_samples.clear();
+    }
+};
+
+} // namespace funasr
+#endif

runtime/onnxruntime/include/com-define.h

@@ -0,0 +1,140 @@
+#pragma once 
+
+namespace funasr {
+#define S_BEGIN  0
+#define S_MIDDLE 1
+#define S_END    2
+#define S_ALL    3
+#define S_ERR    4
+
+#ifndef MODEL_SAMPLE_RATE
+#define MODEL_SAMPLE_RATE 16000
+#endif
+
+// parser option
+#define MODEL_DIR "model-dir"
+#define OFFLINE_MODEL_DIR "model-dir"
+#define ONLINE_MODEL_DIR "online-model-dir"
+#define LM_DIR "lm-dir"
+#define GLOB_BEAM "global-beam"
+#define LAT_BEAM "lattice-beam"
+#define AM_SCALE "am-scale"
+// #define FST_HOTWORD "fst-hotword"
+#define FST_INC_WTS "fst-inc-wts"
+#define VAD_DIR "vad-dir"
+#define PUNC_DIR "punc-dir"
+#define QUANTIZE "quantize"
+#define VAD_QUANT "vad-quant"
+#define PUNC_QUANT "punc-quant"
+#define ASR_MODE "mode"
+
+#define WAV_PATH "wav-path"
+#define WAV_SCP "wav-scp"
+#define TXT_PATH "txt-path"
+#define THREAD_NUM "thread-num"
+#define PORT_ID "port-id"
+#define HOTWORD_SEP " "
+#define AUDIO_FS "audio-fs"
+
+#define MODEL_PARA "Paraformer"
+#define MODEL_SVS "SenseVoiceSmall"
+
+// #define VAD_MODEL_PATH "vad-model"
+// #define VAD_CMVN_PATH "vad-cmvn"
+// #define VAD_CONFIG_PATH "vad-config"
+// #define AM_MODEL_PATH "am-model"
+// #define AM_CMVN_PATH "am-cmvn"
+// #define AM_CONFIG_PATH "am-config"
+// #define PUNC_MODEL_PATH "punc-model"
+// #define PUNC_CONFIG_PATH "punc-config"
+
+#define MODEL_NAME "model.onnx"
+// hotword embedding compile model
+#define MODEL_EB_NAME "model_eb.onnx"
+#define TORCH_MODEL_EB_NAME "model_eb.torchscript"
+#define QUANT_MODEL_NAME "model_quant.onnx"
+#define VAD_CMVN_NAME "am.mvn"
+#define VAD_CONFIG_NAME "config.yaml"
+
+// gpu models
+#define INFER_GPU "gpu"
+#define BATCHSIZE "batch-size"
+#define TORCH_MODEL_NAME "model.torchscript"
+#define TORCH_QUANT_MODEL_NAME "model_quant.torchscript"
+#define BLADE_MODEL_NAME "model_blade.torchscript"
+#define BLADEDISC "bladedisc"
+
+#define AM_CMVN_NAME "am.mvn"
+#define AM_CONFIG_NAME "config.yaml"
+#define LM_CONFIG_NAME "config.yaml"
+#define PUNC_CONFIG_NAME "config.yaml"
+#define MODEL_SEG_DICT "seg_dict"
+#define TOKEN_PATH "tokens.json"
+#define HOTWORD "hotword"
+// #define NN_HOTWORD "nn-hotword"
+
+#define ITN_DIR "itn-dir"
+#define ITN_TAGGER_NAME "zh_itn_tagger.fst"
+#define ITN_VERBALIZER_NAME "zh_itn_verbalizer.fst"
+
+#define ENCODER_NAME "model.onnx"
+#define QUANT_ENCODER_NAME "model_quant.onnx"
+#define DECODER_NAME "decoder.onnx"
+#define QUANT_DECODER_NAME "decoder_quant.onnx"
+
+#define LM_FST_RES "TLG.fst"
+#define LEX_PATH "lexicon.txt"
+
+// vad
+#ifndef VAD_SILENCE_DURATION
+#define VAD_SILENCE_DURATION 800
+#endif
+
+#ifndef VAD_MAX_LEN
+#define VAD_MAX_LEN 15000
+#endif
+
+#ifndef VAD_SPEECH_NOISE_THRES
+#define VAD_SPEECH_NOISE_THRES 0.9
+#endif
+
+#ifndef VAD_LFR_M
+#define VAD_LFR_M 5
+#endif
+
+#ifndef VAD_LFR_N
+#define VAD_LFR_N 1
+#endif
+
+// asr
+#ifndef PARA_LFR_M
+#define PARA_LFR_M 7
+#endif
+
+#ifndef PARA_LFR_N
+#define PARA_LFR_N 6
+#endif
+
+#ifndef ONLINE_STEP
+#define ONLINE_STEP 9600
+#endif
+
+// punc
+#define UNK_CHAR "<unk>"
+#define TOKEN_LEN     20
+
+#define CANDIDATE_NUM   6
+#define UNKNOW_INDEX 0
+#define NOTPUNC  "_"
+#define NOTPUNC_INDEX 1
+#define COMMA_INDEX 2
+#define PERIOD_INDEX 3
+#define QUESTION_INDEX 4
+#define DUN_INDEX 5
+#define CACHE_POP_TRIGGER_LIMIT   200
+
+#define JIEBA_DICT "jieba.c.dict"
+#define JIEBA_USERDICT "jieba_usr_dict"
+#define JIEBA_HMM_MODEL "jieba.hmm"
+
+} // namespace funasr

runtime/onnxruntime/include/funasrruntime.h

@@ -0,0 +1,135 @@
+#pragma once
+#include <map>
+#include <vector>
+#include <unordered_map>
+#ifdef WIN32
+#ifdef _FUNASR_API_EXPORT
+#define  _FUNASRAPI __declspec(dllexport)
+#else
+#define  _FUNASRAPI __declspec(dllimport)
+#endif
+#else
+#define _FUNASRAPI
+#endif
+
+#ifndef _WIN32
+#define FUNASR_CALLBCK_PREFIX __attribute__((__stdcall__))
+#else
+#define FUNASR_CALLBCK_PREFIX __stdcall
+#endif
+
+
+typedef void* FUNASR_HANDLE;
+typedef void* FUNASR_RESULT;
+typedef void* FUNASR_DEC_HANDLE;
+typedef unsigned char FUNASR_BOOL;
+
+#define FUNASR_TRUE 1
+#define FUNASR_FALSE 0
+#define QM_DEFAULT_THREAD_NUM  4
+
+typedef enum
+{
+ RASR_NONE=-1,
+ RASRM_CTC_GREEDY_SEARCH=0,
+ RASRM_CTC_RPEFIX_BEAM_SEARCH = 1,
+ RASRM_ATTENSION_RESCORING = 2,
+}FUNASR_MODE;
+
+typedef enum {
+	FUNASR_MODEL_PADDLE = 0,
+	FUNASR_MODEL_PADDLE_2 = 1,
+	FUNASR_MODEL_K2 = 2,
+	FUNASR_MODEL_PARAFORMER = 3,
+}FUNASR_MODEL_TYPE;
+
+typedef enum {
+	ASR_OFFLINE=0,
+	ASR_ONLINE=1,
+	ASR_TWO_PASS=2,
+}ASR_TYPE;
+
+typedef enum {
+	PUNC_OFFLINE=0,
+	PUNC_ONLINE=1,
+}PUNC_TYPE;
+
+typedef void (* QM_CALLBACK)(int cur_step, int n_total); // n_total: total steps; cur_step: Current Step.
+
+// ASR
+_FUNASRAPI FUNASR_HANDLE  	FunASRInit(std::map<std::string, std::string>& model_path, int thread_num, ASR_TYPE type=ASR_OFFLINE);
+_FUNASRAPI FUNASR_HANDLE  	FunASROnlineInit(FUNASR_HANDLE asr_handle, std::vector<int> chunk_size={5,10,5});
+_FUNASRAPI void         	FunASRReset(FUNASR_HANDLE handle, FUNASR_DEC_HANDLE dec_handle=nullptr);
+
+// buffer
+_FUNASRAPI FUNASR_RESULT	FunASRInferBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len, FUNASR_MODE mode, QM_CALLBACK fn_callback, bool input_finished=true, int sampling_rate=16000, std::string wav_format="pcm");
+// file, support wav & pcm
+_FUNASRAPI FUNASR_RESULT	FunASRInfer(FUNASR_HANDLE handle, const char* sz_filename, FUNASR_MODE mode, QM_CALLBACK fn_callback, int sampling_rate=16000);
+
+_FUNASRAPI const char*	FunASRGetResult(FUNASR_RESULT result,int n_index);
+_FUNASRAPI const char*	FunASRGetStamp(FUNASR_RESULT result);
+_FUNASRAPI const char*	FunASRGetStampSents(FUNASR_RESULT result);
+_FUNASRAPI const char*	FunASRGetTpassResult(FUNASR_RESULT result,int n_index);
+_FUNASRAPI const int64_t	FunASRGetTpassStart(FUNASR_RESULT result);
+_FUNASRAPI const int64_t	FunASRGetTpassEnd(FUNASR_RESULT result);
+_FUNASRAPI const int	FunASRGetRetNumber(FUNASR_RESULT result);
+_FUNASRAPI void			FunASRFreeResult(FUNASR_RESULT result);
+_FUNASRAPI void			FunASRUninit(FUNASR_HANDLE handle);
+_FUNASRAPI const float	FunASRGetRetSnippetTime(FUNASR_RESULT result);
+
+// VAD
+_FUNASRAPI FUNASR_HANDLE  	FsmnVadInit(std::map<std::string, std::string>& model_path, int thread_num);
+_FUNASRAPI FUNASR_HANDLE  	FsmnVadOnlineInit(FUNASR_HANDLE fsmnvad_handle);
+// buffer
+_FUNASRAPI FUNASR_RESULT	FsmnVadInferBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len, QM_CALLBACK fn_callback, bool input_finished=true, int sampling_rate=16000, std::string wav_format="pcm");
+// file, support wav & pcm
+_FUNASRAPI FUNASR_RESULT	FsmnVadInfer(FUNASR_HANDLE handle, const char* sz_filename, QM_CALLBACK fn_callback, int sampling_rate=16000);
+
+_FUNASRAPI std::vector<std::vector<int>>*	FsmnVadGetResult(FUNASR_RESULT result,int n_index);
+_FUNASRAPI void			 	FsmnVadFreeResult(FUNASR_RESULT result);
+_FUNASRAPI void				FsmnVadUninit(FUNASR_HANDLE handle);
+_FUNASRAPI const float		FsmnVadGetRetSnippetTime(FUNASR_RESULT result);
+
+// PUNC
+_FUNASRAPI FUNASR_HANDLE  		CTTransformerInit(std::map<std::string, std::string>& model_path, int thread_num, PUNC_TYPE type=PUNC_OFFLINE);
+_FUNASRAPI FUNASR_RESULT     	CTTransformerInfer(FUNASR_HANDLE handle, const char* sz_sentence, FUNASR_MODE mode, QM_CALLBACK fn_callback, PUNC_TYPE type=PUNC_OFFLINE, FUNASR_RESULT pre_result=nullptr);
+_FUNASRAPI const char* 			CTTransformerGetResult(FUNASR_RESULT result,int n_index);
+_FUNASRAPI void					CTTransformerFreeResult(FUNASR_RESULT result);
+_FUNASRAPI void					CTTransformerUninit(FUNASR_HANDLE handle);
+
+//OfflineStream
+_FUNASRAPI FUNASR_HANDLE  	FunOfflineInit(std::map<std::string, std::string>& model_path, int thread_num, bool use_gpu=false, int batch_size=1);
+_FUNASRAPI void         	FunOfflineReset(FUNASR_HANDLE handle, FUNASR_DEC_HANDLE dec_handle=nullptr);
+// buffer
+_FUNASRAPI FUNASR_RESULT	FunOfflineInferBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len, 
+												  FUNASR_MODE mode, QM_CALLBACK fn_callback, const std::vector<std::vector<float>> &hw_emb, 
+												  int sampling_rate=16000, std::string wav_format="pcm", bool itn=true, FUNASR_DEC_HANDLE dec_handle=nullptr,
+												  std::string svs_lang="auto", bool svs_itn=true);
+// file, support wav & pcm
+_FUNASRAPI FUNASR_RESULT	FunOfflineInfer(FUNASR_HANDLE handle, const char* sz_filename, FUNASR_MODE mode, 
+											QM_CALLBACK fn_callback, const std::vector<std::vector<float>> &hw_emb, 
+											int sampling_rate=16000, bool itn=true, FUNASR_DEC_HANDLE dec_handle=nullptr);
+//#if !defined(__APPLE__)
+_FUNASRAPI const std::vector<std::vector<float>> CompileHotwordEmbedding(FUNASR_HANDLE handle, std::string &hotwords, ASR_TYPE mode=ASR_OFFLINE);
+//#endif
+
+_FUNASRAPI void				FunOfflineUninit(FUNASR_HANDLE handle);
+
+//2passStream
+_FUNASRAPI FUNASR_HANDLE  	FunTpassInit(std::map<std::string, std::string>& model_path, int thread_num);
+_FUNASRAPI FUNASR_HANDLE    FunTpassOnlineInit(FUNASR_HANDLE tpass_handle, std::vector<int> chunk_size={5,10,5});
+// buffer
+_FUNASRAPI FUNASR_RESULT	FunTpassInferBuffer(FUNASR_HANDLE handle, FUNASR_HANDLE online_handle, const char* sz_buf, 
+												int n_len, std::vector<std::vector<std::string>> &punc_cache, bool input_finished=true, 
+												int sampling_rate=16000, std::string wav_format="pcm", ASR_TYPE mode=ASR_TWO_PASS, 
+												const std::vector<std::vector<float>> &hw_emb={{0.0}}, bool itn=true, FUNASR_DEC_HANDLE dec_handle=nullptr,
+												std::string svs_lang="auto", bool svs_itn=true);
+_FUNASRAPI void				FunTpassUninit(FUNASR_HANDLE handle);
+_FUNASRAPI void				FunTpassOnlineUninit(FUNASR_HANDLE handle);
+
+// wfst decoder
+_FUNASRAPI FUNASR_DEC_HANDLE	FunASRWfstDecoderInit(FUNASR_HANDLE handle, int asr_type, float glob_beam, float lat_beam, float am_scale);
+_FUNASRAPI void			FunASRWfstDecoderUninit(FUNASR_DEC_HANDLE handle);
+_FUNASRAPI void			FunWfstDecoderLoadHwsRes(FUNASR_DEC_HANDLE handle, int inc_bias, std::unordered_map<std::string, int> &hws_map);
+_FUNASRAPI void			FunWfstDecoderUnloadHwsRes(FUNASR_DEC_HANDLE handle);
+

runtime/onnxruntime/include/itn-model.h

@@ -0,0 +1,20 @@
+
+#ifndef ITN_MODEL_H
+#define ITN_MODEL_H
+
+#include <string>
+#include <map>
+#include <vector>
+
+namespace funasr {
+class ITNModel {
+  public:
+    virtual ~ITNModel(){};
+    virtual void InitITN(const std::string &itn_tagger, const std::string &itn_verbalizer, int thread_num)=0;
+    virtual std::string Normalize(const std::string& input){return "";};
+};
+
+ITNModel *CreateITNModel(std::map<std::string, std::string>& model_path, int thread_num);
+
+} // namespace funasr
+#endif

runtime/onnxruntime/include/model.h

@@ -0,0 +1,48 @@
+
+#ifndef MODEL_H
+#define MODEL_H
+
+#include <string>
+#include <map>
+#include "funasrruntime.h"
+#include "vocab.h"
+#include "phone-set.h"
+#include "fst/fstlib.h"
+#include "fst/symbol-table.h"
+namespace funasr {
+class Model {
+  public:
+    virtual ~Model(){};
+    virtual void StartUtterance() = 0;
+    virtual void EndUtterance() = 0;
+    virtual void Reset() = 0;
+    virtual string GreedySearch(float* in, int n_len, int64_t token_nums, bool is_stamp=false, std::vector<float> us_alphas={0}, std::vector<float> us_cif_peak={0}){return "";};
+    virtual void InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){};
+    virtual void InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){};
+    virtual void InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, 
+      const std::string &am_config, const std::string &token_file, const std::string &online_token_file, int thread_num){};
+    virtual void InitLm(const std::string &lm_file, const std::string &lm_config, const std::string &lex_file){};
+    virtual void InitFstDecoder(){};
+    virtual std::string Forward(float *din, int len, bool input_finished, const std::vector<std::vector<float>> &hw_emb={{0.0}}, void* wfst_decoder=nullptr){return "";};
+    virtual std::vector<std::string> Forward(float** din, int* len, bool input_finished, const std::vector<std::vector<float>> &hw_emb={{0.0}}, void* wfst_decoder=nullptr, int batch_in=1)
+      {return std::vector<string>();};
+    virtual std::vector<std::string> Forward(float** din, int* len, bool input_finished, std::string svs_lang="auto", bool svs_itn=false, int batch_in=1)
+      {return std::vector<string>();};
+    virtual std::string Rescoring() = 0;
+    virtual void InitHwCompiler(const std::string &hw_model, int thread_num){};
+    virtual void InitSegDict(const std::string &seg_dict_model){};
+    virtual std::vector<std::vector<float>> CompileHotwordEmbedding(std::string &hotwords){return std::vector<std::vector<float>>();};
+    virtual std::string GetLang(){return "";};
+    virtual int GetAsrSampleRate() = 0;
+    virtual void SetBatchSize(int batch_size) {};
+    virtual int GetBatchSize() {return 0;};
+    virtual Vocab* GetVocab() {return nullptr;};
+    virtual Vocab* GetLmVocab() {return nullptr;};
+    virtual PhoneSet* GetPhoneSet() {return nullptr;};
+};
+
+Model *CreateModel(std::map<std::string, std::string>& model_path, int thread_num=1, ASR_TYPE type=ASR_OFFLINE);
+Model *CreateModel(void* asr_handle, std::vector<int> chunk_size);
+
+} // namespace funasr
+#endif

runtime/onnxruntime/include/offline-stream.h

@@ -0,0 +1,41 @@
+#ifndef OFFLINE_STREAM_H
+#define OFFLINE_STREAM_H
+
+#include <memory>
+#include <string>
+#include <map>
+#include "model.h"
+#include "punc-model.h"
+#include "vad-model.h"
+#if !defined(__APPLE__)
+#include "itn-model.h"
+#include "com-define.h"
+#endif
+
+namespace funasr {
+class OfflineStream {
+  public:
+    OfflineStream(std::map<std::string, std::string>& model_path, int thread_num, bool use_gpu=false, int batch_size=1);
+    ~OfflineStream(){};
+
+    std::unique_ptr<VadModel> vad_handle= nullptr;
+    std::unique_ptr<Model> asr_handle= nullptr;
+    std::unique_ptr<PuncModel> punc_handle= nullptr;
+#if !defined(__APPLE__)
+    std::unique_ptr<ITNModel> itn_handle = nullptr;
+#endif
+    bool UseVad(){return use_vad;};
+    bool UsePunc(){return use_punc;}; 
+    bool UseITN(){return use_itn;};
+    std::string GetModelType(){return model_type;};
+    
+  private:
+    bool use_vad=false;
+    bool use_punc=false;
+    bool use_itn=false;
+    std::string model_type = MODEL_PARA;
+};
+
+OfflineStream *CreateOfflineStream(std::map<std::string, std::string>& model_path, int thread_num=1, bool use_gpu=false, int batch_size=1);
+} // namespace funasr
+#endif

runtime/onnxruntime/include/punc-model.h

@@ -0,0 +1,21 @@
+
+#ifndef PUNC_MODEL_H
+#define PUNC_MODEL_H
+
+#include <string>
+#include <map>
+#include <vector>
+#include "funasrruntime.h"
+
+namespace funasr {
+class PuncModel {
+  public:
+    virtual ~PuncModel(){};
+	  virtual void InitPunc(const std::string &punc_model, const std::string &punc_config, const std::string &token_file, int thread_num)=0;
+	  virtual std::string AddPunc(const char* sz_input, std::string language="zh-cn"){return "";};
+	  virtual std::string AddPunc(const char* sz_input, std::vector<std::string>& arr_cache, std::string language="zh-cn"){return "";};
+};
+
+PuncModel *CreatePuncModel(std::map<std::string, std::string>& model_path, int thread_num, PUNC_TYPE type=PUNC_OFFLINE);
+} // namespace funasr
+#endif

runtime/onnxruntime/include/tclap/Arg.h

@@ -0,0 +1,683 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/******************************************************************************
+ *
+ *  file:  Arg.h
+ *
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  Copyright (c) 2004, Michael E. Smoot, Daniel Aarno .
+ *  Copyright (c) 2017 Google Inc.
+ *  All rights reserved.
+ *
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ *  DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************/
+
+
+#ifndef TCLAP_ARGUMENT_H
+#define TCLAP_ARGUMENT_H
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <string>
+#include <vector>
+#include <list>
+#include <iostream>
+#include <iomanip>
+#include <cstdio>
+
+#include <tclap/sstream.h>
+
+#include <tclap/ArgException.h>
+#include <tclap/Visitor.h>
+#include <tclap/CmdLineInterface.h>
+#include <tclap/ArgTraits.h>
+#include <tclap/StandardTraits.h>
+
+namespace TCLAP {
+
+/**
+ * A virtual base class that defines the essential data for all arguments.
+ * This class, or one of its existing children, must be subclassed to do
+ * anything.
+ */
+class Arg
+{
+	private:
+		/**
+		 * Prevent accidental copying.
+		 */
+		Arg(const Arg& rhs);
+
+		/**
+		 * Prevent accidental copying.
+		 */
+		Arg& operator=(const Arg& rhs);
+
+		/**
+		 * Indicates whether the rest of the arguments should be ignored.
+		 */
+		static bool& ignoreRestRef() { static bool ign = false; return ign; }
+
+		/**
+		 * The delimiter that separates an argument flag/name from the
+		 * value.
+		 */
+		static char& delimiterRef() { static char delim = ' '; return delim; }
+
+	protected:
+
+		/**
+		 * The single char flag used to identify the argument.
+		 * This value (preceded by a dash {-}), can be used to identify
+		 * an argument on the command line.  The _flag can be blank,
+		 * in fact this is how unlabeled args work.  Unlabeled args must
+		 * override appropriate functions to get correct handling. Note
+		 * that the _flag does NOT include the dash as part of the flag.
+		 */
+		std::string _flag;
+
+		/**
+		 * A single word namd identifying the argument.
+		 * This value (preceded by two dashed {--}) can also be used
+		 * to identify an argument on the command line.  Note that the
+		 * _name does NOT include the two dashes as part of the _name. The
+		 * _name cannot be blank.
+		 */
+		std::string _name;
+
+		/**
+		 * Description of the argument.
+		 */
+		std::string _description;
+
+		/**
+		 * Indicating whether the argument is required.
+		 */
+		bool _required;
+
+		/**
+		 * Label to be used in usage description.  Normally set to
+		 * "required", but can be changed when necessary.
+		 */
+		std::string _requireLabel;
+
+		/**
+		 * Indicates whether a value is required for the argument.
+		 * Note that the value may be required but the argument/value
+		 * combination may not be, as specified by _required.
+		 */
+		bool _valueRequired;
+
+		/**
+		 * Indicates whether the argument has been set.
+		 * Indicates that a value on the command line has matched the
+		 * name/flag of this argument and the values have been set accordingly.
+		 */
+		bool _alreadySet;
+
+		/**
+		 * A pointer to a visitor object.
+		 * The visitor allows special handling to occur as soon as the
+		 * argument is matched.  This defaults to NULL and should not
+		 * be used unless absolutely necessary.
+		 */
+		Visitor* _visitor;
+
+		/**
+		 * Whether this argument can be ignored, if desired.
+		 */
+		bool _ignoreable;
+
+		/**
+		 * Indicates that the arg was set as part of an XOR and not on the
+		 * command line.
+		 */
+		bool _xorSet;
+
+		bool _acceptsMultipleValues;
+
+		/**
+		 * Performs the special handling described by the Visitor.
+		 */
+		void _checkWithVisitor() const;
+
+		/**
+		 * Primary constructor. YOU (yes you) should NEVER construct an Arg
+		 * directly, this is a base class that is extended by various children
+		 * that are meant to be used.  Use SwitchArg, ValueArg, MultiArg,
+		 * UnlabeledValueArg, or UnlabeledMultiArg instead.
+		 *
+		 * \param flag - The flag identifying the argument.
+		 * \param name - The name identifying the argument.
+		 * \param desc - The description of the argument, used in the usage.
+		 * \param req - Whether the argument is required.
+		 * \param valreq - Whether the a value is required for the argument.
+		 * \param v - The visitor checked by the argument. Defaults to NULL.
+		 */
+ 		Arg( const std::string& flag,
+			 const std::string& name,
+			 const std::string& desc,
+			 bool req,
+			 bool valreq,
+			 Visitor* v = NULL );
+
+	public:
+		/**
+		 * Destructor.
+		 */
+		virtual ~Arg();
+
+		/**
+		 * Adds this to the specified list of Args.
+		 * \param argList - The list to add this to.
+		 */
+		virtual void addToList( std::list<Arg*>& argList ) const;
+
+		/**
+		 * Begin ignoring arguments since the "--" argument was specified.
+		 */
+		static void beginIgnoring() { ignoreRestRef() = true; }
+
+		/**
+		 * Whether to ignore the rest.
+		 */
+		static bool ignoreRest() { return ignoreRestRef(); }
+
+		/**
+		 * The delimiter that separates an argument flag/name from the
+		 * value.
+		 */
+		static char delimiter() { return delimiterRef(); }
+
+		/**
+		 * The char used as a place holder when SwitchArgs are combined.
+		 * Currently set to the bell char (ASCII 7).
+		 */
+		static char blankChar() { return (char)7; }
+
+		/**
+		 * The char that indicates the beginning of a flag.  Defaults to '-', but
+		 * clients can define TCLAP_FLAGSTARTCHAR to override.
+		 */
+#ifndef TCLAP_FLAGSTARTCHAR
+#define TCLAP_FLAGSTARTCHAR '-'
+#endif
+		static char flagStartChar() { return TCLAP_FLAGSTARTCHAR; }
+
+		/**
+		 * The sting that indicates the beginning of a flag.  Defaults to "-", but
+		 * clients can define TCLAP_FLAGSTARTSTRING to override. Should be the same
+		 * as TCLAP_FLAGSTARTCHAR.
+		 */
+#ifndef TCLAP_FLAGSTARTSTRING
+#define TCLAP_FLAGSTARTSTRING "-"
+#endif
+		static const std::string flagStartString() { return TCLAP_FLAGSTARTSTRING; }
+
+		/**
+		 * The sting that indicates the beginning of a name.  Defaults to "--", but
+		 *  clients can define TCLAP_NAMESTARTSTRING to override.
+		 */
+#ifndef TCLAP_NAMESTARTSTRING
+#define TCLAP_NAMESTARTSTRING "--"
+#endif
+		static const std::string nameStartString() { return TCLAP_NAMESTARTSTRING; }
+
+		/**
+		 * The name used to identify the ignore rest argument.
+		 */
+		static const std::string ignoreNameString() { return "ignore_rest"; }
+
+		/**
+		 * Sets the delimiter for all arguments.
+		 * \param c - The character that delimits flags/names from values.
+		 */
+		static void setDelimiter( char c ) { delimiterRef() = c; }
+
+		/**
+		 * Pure virtual method meant to handle the parsing and value assignment
+		 * of the string on the command line.
+		 * \param i - Pointer the the current argument in the list.
+		 * \param args - Mutable list of strings. What is
+		 * passed in from main.
+		 */
+		virtual bool processArg(int *i, std::vector<std::string>& args) = 0;
+
+		/**
+		 * Operator ==.
+		 * Equality operator. Must be virtual to handle unlabeled args.
+		 * \param a - The Arg to be compared to this.
+		 */
+		virtual bool operator==(const Arg& a) const;
+
+		/**
+		 * Returns the argument flag.
+		 */
+		const std::string& getFlag() const;
+
+		/**
+		 * Returns the argument name.
+		 */
+		const std::string& getName() const;
+
+		/**
+		 * Returns the argument description.
+		 */
+		std::string getDescription() const;
+
+		/**
+		 * Indicates whether the argument is required.
+		 */
+		virtual bool isRequired() const;
+
+		/**
+		 * Sets _required to true. This is used by the XorHandler.
+		 * You really have no reason to ever use it.
+		 */
+		void forceRequired();
+
+		/**
+		 * Sets the _alreadySet value to true.  This is used by the XorHandler.
+		 * You really have no reason to ever use it.
+		 */
+		void xorSet();
+
+		/**
+		 * Indicates whether a value must be specified for argument.
+		 */
+		bool isValueRequired() const;
+
+		/**
+		 * Indicates whether the argument has already been set.  Only true
+		 * if the arg has been matched on the command line.
+		 */
+		bool isSet() const;
+
+		/**
+		 * Indicates whether the argument can be ignored, if desired.
+		 */
+		bool isIgnoreable() const;
+
+		/**
+		 * A method that tests whether a string matches this argument.
+		 * This is generally called by the processArg() method.  This
+		 * method could be re-implemented by a child to change how
+		 * arguments are specified on the command line.
+		 * \param s - The string to be compared to the flag/name to determine
+		 * whether the arg matches.
+		 */
+		virtual bool argMatches( const std::string& s ) const;
+
+		/**
+		 * Returns a simple string representation of the argument.
+		 * Primarily for debugging.
+		 */
+		virtual std::string toString() const;
+
+		/**
+		 * Returns a short ID for the usage.
+		 * \param valueId - The value used in the id.
+		 */
+		virtual std::string shortID( const std::string& valueId = "val" ) const;
+
+		/**
+		 * Returns a long ID for the usage.
+		 * \param valueId - The value used in the id.
+		 */
+		virtual std::string longID( const std::string& valueId = "val" ) const;
+
+		/**
+		 * Trims a value off of the flag.
+		 * \param flag - The string from which the flag and value will be
+		 * trimmed. Contains the flag once the value has been trimmed.
+		 * \param value - Where the value trimmed from the string will
+		 * be stored.
+		 */
+		virtual void trimFlag( std::string& flag, std::string& value ) const;
+
+		/**
+		 * Checks whether a given string has blank chars, indicating that
+		 * it is a combined SwitchArg.  If so, return true, otherwise return
+		 * false.
+		 * \param s - string to be checked.
+		 */
+		bool _hasBlanks( const std::string& s ) const;
+
+		/**
+		 * Sets the requireLabel. Used by XorHandler.  You shouldn't ever
+		 * use this.
+		 * \param s - Set the requireLabel to this value.
+		 */
+		void setRequireLabel( const std::string& s );
+
+		/**
+		 * Used for MultiArgs and XorHandler to determine whether args
+		 * can still be set.
+		 */
+		virtual bool allowMore();
+
+		/**
+		 * Use by output classes to determine whether an Arg accepts
+		 * multiple values.
+		 */
+		virtual bool acceptsMultipleValues();
+
+		/**
+		 * Clears the Arg object and allows it to be reused by new
+		 * command lines.
+		 */
+		 virtual void reset();
+};
+
+/**
+ * Typedef of an Arg list iterator.
+ */
+typedef std::list<Arg*>::const_iterator ArgListIterator;
+
+/**
+ * Typedef of an Arg vector iterator.
+ */
+typedef std::vector<Arg*>::const_iterator ArgVectorIterator;
+
+/**
+ * Typedef of a Visitor list iterator.
+ */
+typedef std::list<Visitor*>::const_iterator VisitorListIterator;
+
+/*
+ * Extract a value of type T from it's string representation contained
+ * in strVal. The ValueLike parameter used to select the correct
+ * specialization of ExtractValue depending on the value traits of T.
+ * ValueLike traits use operator>> to assign the value from strVal.
+ */
+template<typename T> void
+ExtractValue(T &destVal, const std::string& strVal, ValueLike vl)
+{
+    static_cast<void>(vl); // Avoid warning about unused vl
+    istringstream is(strVal.c_str());
+
+    int valuesRead = 0;
+    while ( is.good() ) {
+	if ( is.peek() != EOF )
+#ifdef TCLAP_SETBASE_ZERO
+	    is >> std::setbase(0) >> destVal;
+#else
+	    is >> destVal;
+#endif
+	else
+	    break;
+
+	valuesRead++;
+    }
+
+    if ( is.fail() )
+	throw( ArgParseException("Couldn't read argument value "
+				 "from string '" + strVal + "'"));
+
+
+    if ( valuesRead > 1 )
+	throw( ArgParseException("More than one valid value parsed from "
+				 "string '" + strVal + "'"));
+
+}
+
+/*
+ * Extract a value of type T from it's string representation contained
+ * in strVal. The ValueLike parameter used to select the correct
+ * specialization of ExtractValue depending on the value traits of T.
+ * StringLike uses assignment (operator=) to assign from strVal.
+ */
+template<typename T> void
+ExtractValue(T &destVal, const std::string& strVal, StringLike sl)
+{
+    static_cast<void>(sl); // Avoid warning about unused sl
+    SetString(destVal, strVal);
+}
+
+//////////////////////////////////////////////////////////////////////
+//BEGIN Arg.cpp
+//////////////////////////////////////////////////////////////////////
+
+inline Arg::Arg(const std::string& flag,
+         const std::string& name,
+         const std::string& desc,
+         bool req,
+         bool valreq,
+         Visitor* v) :
+  _flag(flag),
+  _name(name),
+  _description(desc),
+  _required(req),
+  _requireLabel("required"),
+  _valueRequired(valreq),
+  _alreadySet(false),
+  _visitor( v ),
+  _ignoreable(true),
+  _xorSet(false),
+  _acceptsMultipleValues(false)
+{
+	if ( _flag.length() > 1 )
+		throw(SpecificationException(
+				"Argument flag can only be one character long", toString() ) );
+
+	if ( _name != ignoreNameString() &&
+		 ( _flag == Arg::flagStartString() ||
+		   _flag == Arg::nameStartString() ||
+		   _flag == " " ) )
+		throw(SpecificationException("Argument flag cannot be either '" +
+							Arg::flagStartString() + "' or '" +
+							Arg::nameStartString() + "' or a space.",
+							toString() ) );
+
+	if ( ( _name.substr( 0, Arg::flagStartString().length() ) == Arg::flagStartString() ) ||
+		 ( _name.substr( 0, Arg::nameStartString().length() ) == Arg::nameStartString() ) ||
+		 ( _name.find( " ", 0 ) != std::string::npos ) )
+		throw(SpecificationException("Argument name begin with either '" +
+							Arg::flagStartString() + "' or '" +
+							Arg::nameStartString() + "' or space.",
+							toString() ) );
+
+}
+
+inline Arg::~Arg() { }
+
+inline std::string Arg::shortID( const std::string& valueId ) const
+{
+	std::string id = "";
+
+	if ( _flag != "" )
+		id = Arg::flagStartString() + _flag;
+	else
+		id = Arg::nameStartString() + _name;
+
+	if ( _valueRequired )
+		id += std::string( 1, Arg::delimiter() ) + "<" + valueId  + ">";
+
+	if ( !_required )
+		id = "[" + id + "]";
+
+	return id;
+}
+
+inline std::string Arg::longID( const std::string& valueId ) const
+{
+	std::string id = "";
+
+	if ( _flag != "" )
+	{
+		id += Arg::flagStartString() + _flag;
+
+		if ( _valueRequired )
+			id += std::string( 1, Arg::delimiter() ) + "<" + valueId + ">";
+
+		id += ",  ";
+	}
+
+	id += Arg::nameStartString() + _name;
+
+	if ( _valueRequired )
+		id += std::string( 1, Arg::delimiter() ) + "<" + valueId + ">";
+
+	return id;
+
+}
+
+inline bool Arg::operator==(const Arg& a) const
+{
+	if ( ( _flag != "" && _flag == a._flag ) || _name == a._name)
+		return true;
+	else
+		return false;
+}
+
+inline std::string Arg::getDescription() const
+{
+	std::string desc = "";
+	if ( _required )
+		desc = "(" + _requireLabel + ")  ";
+
+//	if ( _valueRequired )
+//		desc += "(value required)  ";
+
+	desc += _description;
+	return desc;
+}
+
+inline const std::string& Arg::getFlag() const { return _flag; }
+
+inline const std::string& Arg::getName() const { return _name; }
+
+inline bool Arg::isRequired() const { return _required; }
+
+inline bool Arg::isValueRequired() const { return _valueRequired; }
+
+inline bool Arg::isSet() const
+{
+	if ( _alreadySet && !_xorSet )
+		return true;
+	else
+		return false;
+}
+
+inline bool Arg::isIgnoreable() const { return _ignoreable; }
+
+inline void Arg::setRequireLabel( const std::string& s)
+{
+	_requireLabel = s;
+}
+
+inline bool Arg::argMatches( const std::string& argFlag ) const
+{
+	if ( ( argFlag == Arg::flagStartString() + _flag && _flag != "" ) ||
+	       argFlag == Arg::nameStartString() + _name )
+		return true;
+	else
+		return false;
+}
+
+inline std::string Arg::toString() const
+{
+	std::string s = "";
+
+	if ( _flag != "" )
+		s += Arg::flagStartString() + _flag + " ";
+
+	s += "(" + Arg::nameStartString() + _name + ")";
+
+	return s;
+}
+
+inline void Arg::_checkWithVisitor() const
+{
+	if ( _visitor != NULL )
+		_visitor->visit();
+}
+
+/**
+ * Implementation of trimFlag.
+ */
+inline void Arg::trimFlag(std::string& flag, std::string& value) const
+{
+	int stop = 0;
+	for ( int i = 0; static_cast<unsigned int>(i) < flag.length(); i++ )
+		if ( flag[i] == Arg::delimiter() )
+		{
+			stop = i;
+			break;
+		}
+
+	if ( stop > 1 )
+	{
+		value = flag.substr(stop+1);
+		flag = flag.substr(0,stop);
+	}
+
+}
+
+/**
+ * Implementation of _hasBlanks.
+ */
+inline bool Arg::_hasBlanks( const std::string& s ) const
+{
+	for ( int i = 1; static_cast<unsigned int>(i) < s.length(); i++ )
+		if ( s[i] == Arg::blankChar() )
+			return true;
+
+	return false;
+}
+
+inline void Arg::forceRequired()
+{
+	_required = true;
+}
+
+inline void Arg::xorSet()
+{
+	_alreadySet = true;
+	_xorSet = true;
+}
+
+/**
+ * Overridden by Args that need to added to the end of the list.
+ */
+inline void Arg::addToList( std::list<Arg*>& argList ) const
+{
+	argList.push_front( const_cast<Arg*>(this) );
+}
+
+inline bool Arg::allowMore()
+{
+	return false;
+}
+
+inline bool Arg::acceptsMultipleValues()
+{
+	return _acceptsMultipleValues;
+}
+
+inline void Arg::reset()
+{
+	_xorSet = false;
+	_alreadySet = false;
+}
+
+//////////////////////////////////////////////////////////////////////
+//END Arg.cpp
+//////////////////////////////////////////////////////////////////////
+
+} //namespace TCLAP
+
+#endif
+

runtime/onnxruntime/include/tclap/ArgException.h

@@ -0,0 +1,213 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/****************************************************************************** 
+ * 
+ *  file:  ArgException.h
+ * 
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  Copyright (c) 2017 Google LLC
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+
+#ifndef TCLAP_ARG_EXCEPTION_H
+#define TCLAP_ARG_EXCEPTION_H
+
+#include <string>
+#include <exception>
+
+namespace TCLAP {
+
+/**
+ * A simple class that defines and argument exception.  Should be caught
+ * whenever a CmdLine is created and parsed.
+ */
+class ArgException : public std::exception
+{
+	public:
+	
+		/**
+		 * Constructor.
+		 * \param text - The text of the exception.
+		 * \param id - The text identifying the argument source.
+		 * \param td - Text describing the type of ArgException it is.
+		 * of the exception.
+		 */
+		ArgException( const std::string& text = "undefined exception", 
+					  const std::string& id = "undefined",
+					  const std::string& td = "Generic ArgException")
+			: std::exception(), 
+			  _errorText(text), 
+			  _argId( id ), 
+			  _typeDescription(td)
+		{ } 
+		
+		/**
+		 * Destructor.
+		 */
+		virtual ~ArgException() throw() { }
+
+		/**
+		 * Returns the error text.
+		 */
+		std::string error() const { return ( _errorText ); }
+
+		/**
+		 * Returns the argument id.
+		 */
+		std::string argId() const  
+		{ 
+			if ( _argId == "undefined" )
+				return " ";
+			else
+				return ( "Argument: " + _argId ); 
+		}
+
+		/**
+		 * Returns the arg id and error text. 
+		 */
+		const char* what() const throw() 
+		{
+			static std::string ex; 
+			ex = _argId + " -- " + _errorText;
+			return ex.c_str();
+		}
+
+		/**
+		 * Returns the type of the exception.  Used to explain and distinguish
+		 * between different child exceptions.
+		 */
+		std::string typeDescription() const
+		{
+			return _typeDescription; 
+		}
+
+
+	private:
+
+		/**
+		 * The text of the exception message.
+		 */
+		std::string _errorText;
+
+		/**
+		 * The argument related to this exception.
+		 */
+		std::string _argId;
+
+		/**
+		 * Describes the type of the exception.  Used to distinguish
+		 * between different child exceptions.
+		 */
+		std::string _typeDescription;
+
+};
+
+/**
+ * Thrown from within the child Arg classes when it fails to properly
+ * parse the argument it has been passed.
+ */
+class ArgParseException : public ArgException
+{ 
+	public:
+		/**
+		 * Constructor.
+		 * \param text - The text of the exception.
+		 * \param id - The text identifying the argument source 
+		 * of the exception.
+		 */
+		ArgParseException( const std::string& text = "undefined exception", 
+					       const std::string& id = "undefined" )
+			: ArgException( text, 
+			                id, 
+							std::string( "Exception found while parsing " ) + 
+							std::string( "the value the Arg has been passed." ))
+			{ }
+};
+
+/**
+ * Thrown from CmdLine when the arguments on the command line are not
+ * properly specified, e.g. too many arguments, required argument missing, etc.
+ */
+class CmdLineParseException : public ArgException
+{
+	public:
+		/**
+		 * Constructor.
+		 * \param text - The text of the exception.
+		 * \param id - The text identifying the argument source 
+		 * of the exception.
+		 */
+		CmdLineParseException( const std::string& text = "undefined exception", 
+					           const std::string& id = "undefined" )
+			: ArgException( text, 
+			                id,
+							std::string( "Exception found when the values ") +
+							std::string( "on the command line do not meet ") +
+							std::string( "the requirements of the defined ") +
+							std::string( "Args." ))
+		{ }
+};
+
+/**
+ * Thrown from Arg and CmdLine when an Arg is improperly specified, e.g. 
+ * same flag as another Arg, same name, etc.
+ */
+class SpecificationException : public ArgException
+{
+	public:
+		/**
+		 * Constructor.
+		 * \param text - The text of the exception.
+		 * \param id - The text identifying the argument source 
+		 * of the exception.
+		 */
+		SpecificationException( const std::string& text = "undefined exception",
+					            const std::string& id = "undefined" )
+			: ArgException( text, 
+			                id,
+							std::string("Exception found when an Arg object ")+
+							std::string("is improperly defined by the ") +
+							std::string("developer." )) 
+		{ }
+
+};
+
+/**
+ * Thrown when TCLAP thinks the program should exit.
+ * 
+ * For example after parse error this exception will be thrown (and
+ * normally caught). This allows any resource to be clened properly
+ * before exit.
+ * 
+ * If exception handling is disabled (CmdLine::setExceptionHandling),
+ * this exception will propagate to the call site, allowing the
+ * program to catch it and avoid program termination, or do it's own
+ * cleanup. See for example, https://sourceforge.net/p/tclap/bugs/29.
+ */
+class ExitException {
+public:
+	ExitException(int estat) : _estat(estat) {}
+
+	int getExitStatus() const { return _estat; }
+
+private:
+	int _estat;
+};
+
+} // namespace TCLAP
+
+#endif
+

runtime/onnxruntime/include/tclap/ArgTraits.h

@@ -0,0 +1,122 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/******************************************************************************
+ *
+ *  file:  ArgTraits.h
+ *
+ *  Copyright (c) 2007, Daniel Aarno, Michael E. Smoot .
+ *  Copyright (c) 2017 Google LLC
+ *  All rights reserved.
+ *
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ *  DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************/
+
+// This is an internal tclap file, you should probably not have to
+// include this directly
+
+#ifndef TCLAP_ARGTRAITS_H
+#define TCLAP_ARGTRAITS_H
+
+namespace TCLAP {
+
+// We use two empty structs to get compile type specialization
+// function to work
+
+/**
+ * A value like argument value type is a value that can be set using
+ * operator>>. This is the default value type.
+ */
+struct ValueLike {
+    typedef ValueLike ValueCategory;
+	virtual ~ValueLike() {}
+};
+
+/**
+ * A string like argument value type is a value that can be set using
+ * operator=(string). Useful if the value type contains spaces which
+ * will be broken up into individual tokens by operator>>.
+ */
+struct StringLike {
+	virtual ~StringLike() {}
+};
+
+/**
+ * A class can inherit from this object to make it have string like
+ * traits. This is a compile time thing and does not add any overhead
+ * to the inherenting class.
+ */
+struct StringLikeTrait {
+    typedef StringLike ValueCategory;
+	virtual ~StringLikeTrait() {}
+};
+
+/**
+ * A class can inherit from this object to make it have value like
+ * traits. This is a compile time thing and does not add any overhead
+ * to the inherenting class.
+ */
+struct ValueLikeTrait {
+    typedef ValueLike ValueCategory;
+	virtual ~ValueLikeTrait() {}
+};
+
+/**
+ * Arg traits are used to get compile type specialization when parsing
+ * argument values. Using an ArgTraits you can specify the way that
+ * values gets assigned to any particular type during parsing. The two
+ * supported types are StringLike and ValueLike. ValueLike is the
+ * default and means that operator>> will be used to assign values to
+ * the type.
+ */
+template<typename T>
+class ArgTraits {
+	// This is a bit silly, but what we want to do is:
+	// 1) If there exists a specialization of ArgTraits for type X,
+	// use it.
+	//
+	// 2) If no specialization exists but X has the typename
+	// X::ValueCategory, use the specialization for X::ValueCategory.
+	//
+	// 3) If neither (1) nor (2) defines the trait, use the default
+	// which is ValueLike.
+
+	// This is the "how":
+	//
+	// test<T>(0) (where 0 is the NULL ptr) will match
+	// test(typename C::ValueCategory*) iff type T has the
+	// corresponding typedef. If it does not test(...) will be
+	// matched. This allows us to determine if T::ValueCategory
+	// exists by checking the sizeof for the test function (return
+	// value must have different sizeof).
+	template<typename C> static short test(typename C::ValueCategory*);
+	template<typename C> static long  test(...);
+	static const bool hasTrait = sizeof(test<T>(0)) == sizeof(short);
+
+	template <typename C, bool>
+	struct DefaultArgTrait {
+		typedef ValueLike ValueCategory;
+	};
+
+	template <typename C>
+	struct DefaultArgTrait<C, true> {
+		typedef typename C::ValueCategory ValueCategory;
+ 	};
+
+public:
+	typedef typename DefaultArgTrait<T, hasTrait>::ValueCategory ValueCategory;
+};
+
+} // namespace
+
+#endif
+

runtime/onnxruntime/include/tclap/CmdLine.h

@@ -0,0 +1,657 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/******************************************************************************
+ *
+ *  file:  CmdLine.h
+ *
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  Copyright (c) 2004, Michael E. Smoot, Daniel Aarno.
+ *  All rights reserved.
+ *
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ *  DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************/
+
+#ifndef TCLAP_CMDLINE_H
+#define TCLAP_CMDLINE_H
+
+#include <tclap/SwitchArg.h>
+#include <tclap/MultiSwitchArg.h>
+#include <tclap/UnlabeledValueArg.h>
+#include <tclap/UnlabeledMultiArg.h>
+
+#include <tclap/XorHandler.h>
+#include <tclap/HelpVisitor.h>
+#include <tclap/VersionVisitor.h>
+#include <tclap/IgnoreRestVisitor.h>
+
+#include <tclap/CmdLineOutput.h>
+#include <tclap/StdOutput.h>
+
+#include <tclap/Constraint.h>
+#include <tclap/ValuesConstraint.h>
+
+#include <string>
+#include <vector>
+#include <list>
+#include <iostream>
+#include <iomanip>
+#include <algorithm>
+#include <stdlib.h> // Needed for exit(), which isn't defined in some envs.
+
+namespace TCLAP {
+
+template<typename T> void DelPtr(T ptr)
+{
+	delete ptr;
+}
+
+template<typename C> void ClearContainer(C &c)
+{
+	typedef typename C::value_type value_type;
+	std::for_each(c.begin(), c.end(), DelPtr<value_type>);
+	c.clear();
+}
+
+
+/**
+ * The base class that manages the command line definition and passes
+ * along the parsing to the appropriate Arg classes.
+ */
+class CmdLine : public CmdLineInterface
+{
+	protected:
+
+		/**
+		 * The list of arguments that will be tested against the
+		 * command line.
+		 */
+		std::list<Arg*> _argList;
+
+		/**
+		 * The name of the program.  Set to argv[0].
+		 */
+		std::string _progName;
+
+		/**
+		 * A message used to describe the program.  Used in the usage output.
+		 */
+		std::string _message;
+
+		/**
+		 * The version to be displayed with the --version switch.
+		 */
+		std::string _version;
+
+		/**
+		 * The number of arguments that are required to be present on
+		 * the command line. This is set dynamically, based on the
+		 * Args added to the CmdLine object.
+		 */
+		int _numRequired;
+
+		/**
+		 * The character that is used to separate the argument flag/name
+		 * from the value.  Defaults to ' ' (space).
+		 */
+		char _delimiter;
+
+		/**
+		 * The handler that manages xoring lists of args.
+		 */
+		XorHandler _xorHandler;
+
+		/**
+		 * A list of Args to be explicitly deleted when the destructor
+		 * is called.  At the moment, this only includes the three default
+		 * Args.
+		 */
+		std::list<Arg*> _argDeleteOnExitList;
+
+		/**
+		 * A list of Visitors to be explicitly deleted when the destructor
+		 * is called.  At the moment, these are the Visitors created for the
+		 * default Args.
+		 */
+		std::list<Visitor*> _visitorDeleteOnExitList;
+
+		/**
+		 * Object that handles all output for the CmdLine.
+		 */
+		CmdLineOutput* _output;
+
+		/**
+		 * Should CmdLine handle parsing exceptions internally?
+		 */
+		bool _handleExceptions;
+
+		/**
+		 * Throws an exception listing the missing args.
+		 */
+		void missingArgsException();
+
+		/**
+		 * Checks whether a name/flag string matches entirely matches
+		 * the Arg::blankChar.  Used when multiple switches are combined
+		 * into a single argument.
+		 * \param s - The message to be used in the usage.
+		 */
+		bool _emptyCombined(const std::string& s);
+
+		/**
+		 * Perform a delete ptr; operation on ptr when this object is deleted.
+		 */
+		void deleteOnExit(Arg* ptr);
+
+		/**
+		 * Perform a delete ptr; operation on ptr when this object is deleted.
+		 */
+		void deleteOnExit(Visitor* ptr);
+
+private:
+
+		/**
+		 * Prevent accidental copying.
+		 */
+		CmdLine(const CmdLine& rhs);
+		CmdLine& operator=(const CmdLine& rhs);
+
+		/**
+		 * Encapsulates the code common to the constructors
+		 * (which is all of it).
+		 */
+		void _constructor();
+
+
+		/**
+		 * Is set to true when a user sets the output object. We use this so
+		 * that we don't delete objects that are created outside of this lib.
+		 */
+		bool _userSetOutput;
+
+		/**
+		 * Whether or not to automatically create help and version switches.
+		 */
+		bool _helpAndVersion;
+
+		/**
+		 * Whether or not to ignore unmatched args.
+		 */
+		bool _ignoreUnmatched;
+
+	public:
+
+		/**
+		 * Command line constructor. Defines how the arguments will be
+		 * parsed.
+		 * \param message - The message to be used in the usage
+		 * output.
+		 * \param delimiter - The character that is used to separate
+		 * the argument flag/name from the value.  Defaults to ' ' (space).
+		 * \param version - The version number to be used in the
+		 * --version switch.
+		 * \param helpAndVersion - Whether or not to create the Help and
+		 * Version switches. Defaults to true.
+		 */
+		CmdLine(const std::string& message,
+				const char delimiter = ' ',
+				const std::string& version = "none",
+				bool helpAndVersion = true);
+
+		/**
+		 * Deletes any resources allocated by a CmdLine object.
+		 */
+		virtual ~CmdLine();
+
+		/**
+		 * Adds an argument to the list of arguments to be parsed.
+		 * \param a - Argument to be added.
+		 */
+		void add( Arg& a );
+
+		/**
+		 * An alternative add.  Functionally identical.
+		 * \param a - Argument to be added.
+		 */
+		void add( Arg* a );
+
+		/**
+		 * Add two Args that will be xor'd.  If this method is used, add does
+		 * not need to be called.
+		 * \param a - Argument to be added and xor'd.
+		 * \param b - Argument to be added and xor'd.
+		 */
+		void xorAdd( Arg& a, Arg& b );
+
+		/**
+		 * Add a list of Args that will be xor'd.  If this method is used,
+		 * add does not need to be called.
+		 * \param xors - List of Args to be added and xor'd.
+		 */
+		void xorAdd( const std::vector<Arg*>& xors );
+
+		/**
+		 * Parses the command line.
+		 * \param argc - Number of arguments.
+		 * \param argv - Array of arguments.
+		 */
+		void parse(int argc, const char * const * argv);
+
+		/**
+		 * Parses the command line.
+		 * \param args - A vector of strings representing the args.
+		 * args[0] is still the program name.
+		 */
+		void parse(std::vector<std::string>& args);
+
+		/**
+		 *
+		 */
+		CmdLineOutput* getOutput();
+
+		/**
+		 *
+		 */
+		void setOutput(CmdLineOutput* co);
+
+		/**
+		 *
+		 */
+		std::string& getVersion();
+
+		/**
+		 *
+		 */
+		std::string& getProgramName();
+
+		/**
+		 *
+		 */
+		std::list<Arg*>& getArgList();
+
+		/**
+		 *
+		 */
+		XorHandler& getXorHandler();
+
+		/**
+		 *
+		 */
+		char getDelimiter();
+
+		/**
+		 *
+		 */
+		std::string& getMessage();
+
+		/**
+		 *
+		 */
+		bool hasHelpAndVersion();
+
+		/**
+		 * Disables or enables CmdLine's internal parsing exception handling.
+		 *
+		 * @param state Should CmdLine handle parsing exceptions internally?
+		 */
+		void setExceptionHandling(const bool state);
+
+		/**
+		 * Returns the current state of the internal exception handling.
+		 *
+		 * @retval true Parsing exceptions are handled internally.
+		 * @retval false Parsing exceptions are propagated to the caller.
+		 */
+		bool getExceptionHandling() const;
+
+		/**
+		 * Allows the CmdLine object to be reused.
+		 */
+		void reset();
+
+		/**
+		 * Allows unmatched args to be ignored. By default false.
+		 * 
+		 * @param ignore If true the cmdline will ignore any unmatched args
+		 * and if false it will behave as normal.
+		 */
+		void ignoreUnmatched(const bool ignore);
+};
+
+
+///////////////////////////////////////////////////////////////////////////////
+//Begin CmdLine.cpp
+///////////////////////////////////////////////////////////////////////////////
+
+inline CmdLine::CmdLine(const std::string& m,
+                        char delim,
+                        const std::string& v,
+                        bool help )
+    :
+  _argList(std::list<Arg*>()),
+  _progName("not_set_yet"),
+  _message(m),
+  _version(v),
+  _numRequired(0),
+  _delimiter(delim),
+  _xorHandler(XorHandler()),
+  _argDeleteOnExitList(std::list<Arg*>()),
+  _visitorDeleteOnExitList(std::list<Visitor*>()),
+  _output(0),
+  _handleExceptions(true),
+  _userSetOutput(false),
+  _helpAndVersion(help),
+  _ignoreUnmatched(false)
+{
+	_constructor();
+}
+
+inline CmdLine::~CmdLine()
+{
+	ClearContainer(_argDeleteOnExitList);
+	ClearContainer(_visitorDeleteOnExitList);
+
+	if ( !_userSetOutput ) {
+		delete _output;
+		_output = 0;
+	}
+}
+
+inline void CmdLine::_constructor()
+{
+	_output = new StdOutput;
+
+	Arg::setDelimiter( _delimiter );
+
+	Visitor* v;
+
+	if ( _helpAndVersion )
+	{
+		v = new HelpVisitor( this, &_output );
+		SwitchArg* help = new SwitchArg("h","help",
+		                      "Displays usage information and exits.",
+		                      false, v);
+		add( help );
+		deleteOnExit(help);
+		deleteOnExit(v);
+
+		v = new VersionVisitor( this, &_output );
+		SwitchArg* vers = new SwitchArg("","version",
+		                      "Displays version information and exits.",
+		                      false, v);
+		add( vers );
+		deleteOnExit(vers);
+		deleteOnExit(v);
+	}
+
+	v = new IgnoreRestVisitor();
+	SwitchArg* ignore  = new SwitchArg(Arg::flagStartString(),
+	          Arg::ignoreNameString(),
+	          "Ignores the rest of the labeled arguments following this flag.",
+	          false, v);
+	add( ignore );
+	deleteOnExit(ignore);
+	deleteOnExit(v);
+}
+
+inline void CmdLine::xorAdd( const std::vector<Arg*>& ors )
+{
+	_xorHandler.add( ors );
+
+	for (ArgVectorIterator it = ors.begin(); it != ors.end(); it++)
+	{
+		(*it)->forceRequired();
+		(*it)->setRequireLabel( "OR required" );
+		add( *it );
+	}
+}
+
+inline void CmdLine::xorAdd( Arg& a, Arg& b )
+{
+	std::vector<Arg*> ors;
+	ors.push_back( &a );
+	ors.push_back( &b );
+	xorAdd( ors );
+}
+
+inline void CmdLine::add( Arg& a )
+{
+	add( &a );
+}
+
+inline void CmdLine::add( Arg* a )
+{
+	for( ArgListIterator it = _argList.begin(); it != _argList.end(); it++ )
+		if ( *a == *(*it) )
+			throw( SpecificationException(
+			        "Argument with same flag/name already exists!",
+			        a->longID() ) );
+
+	a->addToList( _argList );
+
+	if ( a->isRequired() )
+		_numRequired++;
+}
+
+
+inline void CmdLine::parse(int argc, const char * const * argv)
+{
+		// this step is necessary so that we have easy access to
+		// mutable strings.
+		std::vector<std::string> args;
+		for (int i = 0; i < argc; i++)
+			args.push_back(argv[i]);
+
+		parse(args);
+}
+
+inline void CmdLine::parse(std::vector<std::string>& args)
+{
+	bool shouldExit = false;
+	int estat = 0;
+	try {
+        if (args.empty()) {
+            // https://sourceforge.net/p/tclap/bugs/30/
+            throw CmdLineParseException("The args vector must not be empty, "
+                                        "the first entry should contain the "
+                                        "program's name.");
+        }
+
+		_progName = args.front();
+		args.erase(args.begin());
+
+		int requiredCount = 0;
+
+		for (int i = 0; static_cast<unsigned int>(i) < args.size(); i++) 
+		{
+			bool matched = false;
+			for (ArgListIterator it = _argList.begin();
+			     it != _argList.end(); it++) {
+				if ( (*it)->processArg( &i, args ) )
+				{
+					requiredCount += _xorHandler.check( *it );
+					matched = true;
+					break;
+				}
+			}
+
+			// checks to see if the argument is an empty combined
+			// switch and if so, then we've actually matched it
+			if ( !matched && _emptyCombined( args[i] ) )
+				matched = true;
+
+			if ( !matched && !Arg::ignoreRest() && !_ignoreUnmatched)
+				throw(CmdLineParseException("Couldn't find match "
+				                            "for argument",
+				                            args[i]));
+		}
+
+		if ( requiredCount < _numRequired )
+			missingArgsException();
+
+		if ( requiredCount > _numRequired )
+			throw(CmdLineParseException("Too many arguments!"));
+
+	} catch ( ArgException& e ) {
+		// If we're not handling the exceptions, rethrow.
+		if ( !_handleExceptions) {
+			throw;
+		}
+
+		try {
+			_output->failure(*this,e);
+		} catch ( ExitException &ee ) {
+			estat = ee.getExitStatus();
+			shouldExit = true;
+		}
+	} catch (ExitException &ee) {
+		// If we're not handling the exceptions, rethrow.
+		if ( !_handleExceptions) {
+			throw;
+		}
+
+		estat = ee.getExitStatus();
+		shouldExit = true;
+	}
+
+	if (shouldExit)
+		exit(estat);
+}
+
+inline bool CmdLine::_emptyCombined(const std::string& s)
+{
+	if ( s.length() > 0 && s[0] != Arg::flagStartChar() )
+		return false;
+
+	for ( int i = 1; static_cast<unsigned int>(i) < s.length(); i++ )
+		if ( s[i] != Arg::blankChar() )
+			return false;
+
+	return true;
+}
+
+inline void CmdLine::missingArgsException()
+{
+		int count = 0;
+
+		std::string missingArgList;
+		for (ArgListIterator it = _argList.begin(); it != _argList.end(); it++)
+		{
+			if ( (*it)->isRequired() && !(*it)->isSet() )
+			{
+				missingArgList += (*it)->getName();
+				missingArgList += ", ";
+				count++;
+			}
+		}
+		missingArgList = missingArgList.substr(0,missingArgList.length()-2);
+
+		std::string msg;
+		if ( count > 1 )
+			msg = "Required arguments missing: ";
+		else
+			msg = "Required argument missing: ";
+
+		msg += missingArgList;
+
+		throw(CmdLineParseException(msg));
+}
+
+inline void CmdLine::deleteOnExit(Arg* ptr)
+{
+	_argDeleteOnExitList.push_back(ptr);
+}
+
+inline void CmdLine::deleteOnExit(Visitor* ptr)
+{
+	_visitorDeleteOnExitList.push_back(ptr);
+}
+
+inline CmdLineOutput* CmdLine::getOutput()
+{
+	return _output;
+}
+
+inline void CmdLine::setOutput(CmdLineOutput* co)
+{
+	if ( !_userSetOutput )
+		delete _output;
+	_userSetOutput = true;
+	_output = co;
+}
+
+inline std::string& CmdLine::getVersion()
+{
+	return _version;
+}
+
+inline std::string& CmdLine::getProgramName()
+{
+	return _progName;
+}
+
+inline std::list<Arg*>& CmdLine::getArgList()
+{
+	return _argList;
+}
+
+inline XorHandler& CmdLine::getXorHandler()
+{
+	return _xorHandler;
+}
+
+inline char CmdLine::getDelimiter()
+{
+	return _delimiter;
+}
+
+inline std::string& CmdLine::getMessage()
+{
+	return _message;
+}
+
+inline bool CmdLine::hasHelpAndVersion()
+{
+	return _helpAndVersion;
+}
+
+inline void CmdLine::setExceptionHandling(const bool state)
+{
+	_handleExceptions = state;
+}
+
+inline bool CmdLine::getExceptionHandling() const
+{
+	return _handleExceptions;
+}
+
+inline void CmdLine::reset()
+{
+	for( ArgListIterator it = _argList.begin(); it != _argList.end(); it++ )
+		(*it)->reset();
+	
+	_progName.clear();
+}
+
+inline void CmdLine::ignoreUnmatched(const bool ignore)
+{
+	_ignoreUnmatched = ignore;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//End CmdLine.cpp
+///////////////////////////////////////////////////////////////////////////////
+
+
+
+} //namespace TCLAP
+#endif

runtime/onnxruntime/include/tclap/CmdLineInterface.h

@@ -0,0 +1,153 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+
+/****************************************************************************** 
+ * 
+ *  file:  CmdLineInterface.h
+ * 
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  Copyright (c) 2004, Michael E. Smoot, Daniel Aarno.
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ *
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+#ifndef TCLAP_COMMANDLINE_INTERFACE_H
+#define TCLAP_COMMANDLINE_INTERFACE_H
+
+#include <string>
+#include <vector>
+#include <list>
+#include <iostream>
+#include <algorithm>
+
+
+namespace TCLAP {
+     
+class Arg;
+class CmdLineOutput;
+class XorHandler;
+
+/**
+ * The base class that manages the command line definition and passes
+ * along the parsing to the appropriate Arg classes.
+ */
+class CmdLineInterface
+{
+	public:
+
+		/**
+		 * Destructor
+		 */
+		virtual ~CmdLineInterface() {}
+
+		/**
+		 * Adds an argument to the list of arguments to be parsed.
+		 * \param a - Argument to be added. 
+		 */
+		virtual void add( Arg& a )=0;
+
+		/**
+		 * An alternative add.  Functionally identical.
+		 * \param a - Argument to be added. 
+		 */
+		virtual void add( Arg* a )=0;
+
+		/**
+		 * Add two Args that will be xor'd.  
+		 * If this method is used, add does
+		 * not need to be called.
+		 * \param a - Argument to be added and xor'd. 
+		 * \param b - Argument to be added and xor'd. 
+		 */
+		virtual void xorAdd( Arg& a, Arg& b )=0;
+
+		/**
+		 * Add a list of Args that will be xor'd.  If this method is used, 
+		 * add does not need to be called.
+		 * \param xors - List of Args to be added and xor'd. 
+		 */
+		virtual void xorAdd( const std::vector<Arg*>& xors )=0;
+
+		/**
+		 * Parses the command line.
+		 * \param argc - Number of arguments.
+		 * \param argv - Array of arguments.
+		 */
+		virtual void parse(int argc, const char * const * argv)=0;
+
+        /**
+         * Parses the command line.
+         * \param args - A vector of strings representing the args. 
+         * args[0] is still the program name.
+         */
+        void parse(std::vector<std::string>& args);
+
+		/**
+		 * Returns the CmdLineOutput object.
+		 */
+		virtual CmdLineOutput* getOutput()=0;
+
+		/**
+		 * \param co - CmdLineOutput object that we want to use instead. 
+		 */
+		virtual void setOutput(CmdLineOutput* co)=0;
+
+		/**
+		 * Returns the version string.
+		 */
+		virtual std::string& getVersion()=0;
+
+		/**
+		 * Returns the program name string.
+		 */
+		virtual std::string& getProgramName()=0;
+
+		/**
+		 * Returns the argList. 
+		 */
+		virtual std::list<Arg*>& getArgList()=0;
+
+		/**
+		 * Returns the XorHandler. 
+		 */
+		virtual XorHandler& getXorHandler()=0;
+
+		/**
+		 * Returns the delimiter string.
+		 */
+		virtual char getDelimiter()=0;
+
+		/**
+		 * Returns the message string.
+		 */
+		virtual std::string& getMessage()=0;
+
+		/**
+		 * Indicates whether or not the help and version switches were created
+		 * automatically.
+		 */
+		virtual bool hasHelpAndVersion()=0;
+
+		/** 
+		 * Resets the instance as if it had just been constructed so that the
+		 * instance can be reused. 
+		 */
+		virtual void reset()=0;
+};
+
+} //namespace
+
+
+#endif 

runtime/onnxruntime/include/tclap/CmdLineOutput.h

@@ -0,0 +1,77 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+
+
+/****************************************************************************** 
+ * 
+ *  file:  CmdLineOutput.h
+ * 
+ *  Copyright (c) 2004, Michael E. Smoot
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+#ifndef TCLAP_CMDLINEOUTPUT_H
+#define TCLAP_CMDLINEOUTPUT_H
+
+#include <string>
+#include <vector>
+#include <list>
+#include <iostream>
+#include <iomanip>
+#include <algorithm>
+
+namespace TCLAP {
+
+class CmdLineInterface;
+class ArgException;
+
+/**
+ * The interface that any output object must implement.
+ */
+class CmdLineOutput 
+{
+
+	public:
+
+		/**
+		 * Virtual destructor.
+		 */
+		virtual ~CmdLineOutput() {}
+
+		/**
+		 * Generates some sort of output for the USAGE. 
+		 * \param c - The CmdLine object the output is generated for. 
+		 */
+		virtual void usage(CmdLineInterface& c)=0;
+
+		/**
+		 * Generates some sort of output for the version. 
+		 * \param c - The CmdLine object the output is generated for. 
+		 */
+		virtual void version(CmdLineInterface& c)=0;
+
+		/**
+		 * Generates some sort of output for a failure. 
+		 * \param c - The CmdLine object the output is generated for. 
+		 * \param e - The ArgException that caused the failure. 
+		 */
+		virtual void failure( CmdLineInterface& c, 
+						      ArgException& e )=0;
+
+};
+
+} //namespace TCLAP
+#endif 

runtime/onnxruntime/include/tclap/Constraint.h

@@ -0,0 +1,78 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+
+/******************************************************************************
+ *
+ *  file:  Constraint.h
+ *
+ *  Copyright (c) 2005, Michael E. Smoot
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ *
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ *  DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************/
+
+#ifndef TCLAP_CONSTRAINT_H
+#define TCLAP_CONSTRAINT_H
+
+#include <string>
+#include <vector>
+#include <list>
+#include <iostream>
+#include <iomanip>
+#include <algorithm>
+#include <stdexcept>
+
+namespace TCLAP {
+
+/**
+ * The interface that defines the interaction between the Arg and Constraint.
+ */
+template<class T>
+class Constraint
+{
+
+	public:
+		/**
+		 * Returns a description of the Constraint.
+		 */
+		virtual std::string description() const =0;
+
+		/**
+		 * Returns the short ID for the Constraint.
+		 */
+		virtual std::string shortID() const =0;
+
+		/**
+		 * The method used to verify that the value parsed from the command
+		 * line meets the constraint.
+		 * \param value - The value that will be checked.
+		 */
+		virtual bool check(const T& value) const =0;
+
+		/**
+		 * Destructor.
+		 * Silences warnings about Constraint being a base class with virtual
+		 * functions but without a virtual destructor.
+		 */
+		virtual ~Constraint() { ; }
+
+		static std::string shortID(Constraint<T> *constraint) {
+		  if (!constraint)
+		    throw std::logic_error("Cannot create a ValueArg with a NULL constraint");
+		  return constraint->shortID();
+		}
+};
+
+} //namespace TCLAP
+#endif

runtime/onnxruntime/include/tclap/DocBookOutput.h

@@ -0,0 +1,303 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/****************************************************************************** 
+ * 
+ *  file:  DocBookOutput.h
+ * 
+ *  Copyright (c) 2004, Michael E. Smoot
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+#ifndef TCLAP_DOCBOOKOUTPUT_H
+#define TCLAP_DOCBOOKOUTPUT_H
+
+#include <string>
+#include <vector>
+#include <list>
+#include <iostream>
+#include <algorithm>
+
+#include <tclap/CmdLineInterface.h>
+#include <tclap/CmdLineOutput.h>
+#include <tclap/XorHandler.h>
+#include <tclap/Arg.h>
+
+namespace TCLAP {
+
+/**
+ * A class that generates DocBook output for usage() method for the 
+ * given CmdLine and its Args.
+ */
+class DocBookOutput : public CmdLineOutput
+{
+
+	public:
+
+		/**
+		 * Prints the usage to stdout.  Can be overridden to 
+		 * produce alternative behavior.
+		 * \param c - The CmdLine object the output is generated for. 
+		 */
+		virtual void usage(CmdLineInterface& c);
+
+		/**
+		 * Prints the version to stdout. Can be overridden 
+		 * to produce alternative behavior.
+		 * \param c - The CmdLine object the output is generated for. 
+		 */
+		virtual void version(CmdLineInterface& c);
+
+		/**
+		 * Prints (to stderr) an error message, short usage 
+		 * Can be overridden to produce alternative behavior.
+		 * \param c - The CmdLine object the output is generated for. 
+		 * \param e - The ArgException that caused the failure. 
+		 */
+		virtual void failure(CmdLineInterface& c, 
+						     ArgException& e );
+
+	    DocBookOutput() : theDelimiter('=') {}
+	protected:
+
+		/**
+		 * Substitutes the char r for string x in string s.
+		 * \param s - The string to operate on. 
+		 * \param r - The char to replace. 
+		 * \param x - What to replace r with. 
+		 */
+		void substituteSpecialChars( std::string& s, char r, std::string& x );
+		void removeChar( std::string& s, char r);
+		void basename( std::string& s );
+
+		void printShortArg(Arg* it);
+		void printLongArg(Arg* it);
+
+		char theDelimiter;
+};
+
+
+inline void DocBookOutput::version(CmdLineInterface& _cmd) 
+{ 
+	std::cout << _cmd.getVersion() << std::endl;
+}
+
+inline void DocBookOutput::usage(CmdLineInterface& _cmd ) 
+{
+	std::list<Arg*> argList = _cmd.getArgList();
+	std::string progName = _cmd.getProgramName();
+	std::string xversion = _cmd.getVersion();
+	theDelimiter = _cmd.getDelimiter();
+	XorHandler xorHandler = _cmd.getXorHandler();
+	const std::vector< std::vector<Arg*> > xorList = xorHandler.getXorList();
+	basename(progName);
+
+	std::cout << "<?xml version='1.0'?>" << std::endl;
+	std::cout << "<!DOCTYPE refentry PUBLIC \"-//OASIS//DTD DocBook XML V4.2//EN\"" << std::endl;
+	std::cout << "\t\"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd\">" << std::endl << std::endl;
+
+	std::cout << "<refentry>" << std::endl;
+
+	std::cout << "<refmeta>" << std::endl;
+	std::cout << "<refentrytitle>" << progName << "</refentrytitle>" << std::endl;
+	std::cout << "<manvolnum>1</manvolnum>" << std::endl;
+	std::cout << "</refmeta>" << std::endl;
+
+	std::cout << "<refnamediv>" << std::endl;
+	std::cout << "<refname>" << progName << "</refname>" << std::endl;
+	std::cout << "<refpurpose>" << _cmd.getMessage() << "</refpurpose>" << std::endl;
+	std::cout << "</refnamediv>" << std::endl;
+
+	std::cout << "<refsynopsisdiv>" << std::endl;
+	std::cout << "<cmdsynopsis>" << std::endl;
+
+	std::cout << "<command>" << progName << "</command>" << std::endl;
+
+	// xor
+	for ( int i = 0; (unsigned int)i < xorList.size(); i++ )
+	{
+		std::cout << "<group choice='req'>" << std::endl;
+		for ( ArgVectorIterator it = xorList[i].begin(); 
+						it != xorList[i].end(); it++ )
+			printShortArg((*it));
+
+		std::cout << "</group>" << std::endl;
+	}
+
+	// rest of args
+	for (ArgListIterator it = argList.begin(); it != argList.end(); it++)
+		if ( !xorHandler.contains( (*it) ) )
+			printShortArg((*it));
+
+ 	std::cout << "</cmdsynopsis>" << std::endl;
+	std::cout << "</refsynopsisdiv>" << std::endl;
+
+	std::cout << "<refsect1>" << std::endl;
+	std::cout << "<title>Description</title>" << std::endl;
+	std::cout << "<para>" << std::endl;
+	std::cout << _cmd.getMessage() << std::endl; 
+	std::cout << "</para>" << std::endl;
+	std::cout << "</refsect1>" << std::endl;
+
+	std::cout << "<refsect1>" << std::endl;
+	std::cout << "<title>Options</title>" << std::endl;
+
+	std::cout << "<variablelist>" << std::endl;
+	
+	for (ArgListIterator it = argList.begin(); it != argList.end(); it++)
+		printLongArg((*it));
+
+	std::cout << "</variablelist>" << std::endl;
+	std::cout << "</refsect1>" << std::endl;
+
+	std::cout << "<refsect1>" << std::endl;
+	std::cout << "<title>Version</title>" << std::endl;
+	std::cout << "<para>" << std::endl;
+	std::cout << xversion << std::endl; 
+	std::cout << "</para>" << std::endl;
+	std::cout << "</refsect1>" << std::endl;
+	
+	std::cout << "</refentry>" << std::endl;
+
+}
+
+inline void DocBookOutput::failure( CmdLineInterface& _cmd,
+				    ArgException& e ) 
+{ 
+	static_cast<void>(_cmd); // unused
+	std::cout << e.what() << std::endl;
+	throw ExitException(1);
+}
+
+inline void DocBookOutput::substituteSpecialChars( std::string& s,
+				                                   char r,
+												   std::string& x )
+{
+	size_t p;
+	while ( (p = s.find_first_of(r)) != std::string::npos )
+	{
+		s.erase(p,1);
+		s.insert(p,x);
+	}
+}
+
+inline void DocBookOutput::removeChar( std::string& s, char r)
+{
+	size_t p;
+	while ( (p = s.find_first_of(r)) != std::string::npos )
+	{
+		s.erase(p,1);
+	}
+}
+
+inline void DocBookOutput::basename( std::string& s )
+{
+	size_t p = s.find_last_of('/');
+	if ( p != std::string::npos )
+	{
+		s.erase(0, p + 1);
+	}
+}
+
+inline void DocBookOutput::printShortArg(Arg* a)
+{
+	std::string lt = "&lt;"; 
+	std::string gt = "&gt;"; 
+
+	std::string id = a->shortID();
+	substituteSpecialChars(id,'<',lt);
+	substituteSpecialChars(id,'>',gt);
+	removeChar(id,'[');
+	removeChar(id,']');
+	
+	std::string choice = "opt";
+	if ( a->isRequired() )
+		choice = "plain";
+
+	std::cout << "<arg choice='" << choice << '\'';
+	if ( a->acceptsMultipleValues() )
+		std::cout << " rep='repeat'";
+
+
+	std::cout << '>';
+	if ( !a->getFlag().empty() )
+		std::cout << a->flagStartChar() << a->getFlag();
+	else
+		std::cout << a->nameStartString() << a->getName();
+	if ( a->isValueRequired() )
+	{
+		std::string arg = a->shortID();
+		removeChar(arg,'[');
+		removeChar(arg,']');
+		removeChar(arg,'<');
+		removeChar(arg,'>');
+		removeChar(arg,'.');
+		arg.erase(0, arg.find_last_of(theDelimiter) + 1);
+		std::cout << theDelimiter;
+		std::cout << "<replaceable>" << arg << "</replaceable>";
+	}
+	std::cout << "</arg>" << std::endl;
+
+}
+
+inline void DocBookOutput::printLongArg(Arg* a)
+{
+	std::string lt = "&lt;"; 
+	std::string gt = "&gt;"; 
+
+	std::string desc = a->getDescription();
+	substituteSpecialChars(desc,'<',lt);
+	substituteSpecialChars(desc,'>',gt);
+
+	std::cout << "<varlistentry>" << std::endl;
+
+	if ( !a->getFlag().empty() )
+	{
+		std::cout << "<term>" << std::endl;
+		std::cout << "<option>";
+		std::cout << a->flagStartChar() << a->getFlag();
+		std::cout << "</option>" << std::endl;
+		std::cout << "</term>" << std::endl;
+	}
+
+	std::cout << "<term>" << std::endl;
+	std::cout << "<option>";
+	std::cout << a->nameStartString() << a->getName();
+	if ( a->isValueRequired() )
+	{
+		std::string arg = a->shortID();
+		removeChar(arg,'[');
+		removeChar(arg,']');
+		removeChar(arg,'<');
+		removeChar(arg,'>');
+		removeChar(arg,'.');
+		arg.erase(0, arg.find_last_of(theDelimiter) + 1);
+		std::cout << theDelimiter;
+		std::cout << "<replaceable>" << arg << "</replaceable>";
+	}
+	std::cout << "</option>" << std::endl;
+	std::cout << "</term>" << std::endl;
+
+	std::cout << "<listitem>" << std::endl;
+	std::cout << "<para>" << std::endl;
+	std::cout << desc << std::endl;
+	std::cout << "</para>" << std::endl;
+	std::cout << "</listitem>" << std::endl;
+
+	std::cout << "</varlistentry>" << std::endl;
+}
+
+} //namespace TCLAP
+#endif 

runtime/onnxruntime/include/tclap/HelpVisitor.h

@@ -0,0 +1,78 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+
+/****************************************************************************** 
+ * 
+ *  file:  HelpVisitor.h
+ * 
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+#ifndef TCLAP_HELP_VISITOR_H
+#define TCLAP_HELP_VISITOR_H
+
+#include <tclap/CmdLineInterface.h>
+#include <tclap/CmdLineOutput.h>
+#include <tclap/Visitor.h>
+
+namespace TCLAP {
+
+/**
+ * A Visitor object that calls the usage method of the given CmdLineOutput
+ * object for the specified CmdLine object.
+ */
+class HelpVisitor: public Visitor
+{
+	private:
+		/**
+		 * Prevent accidental copying.
+		 */
+		HelpVisitor(const HelpVisitor& rhs);
+		HelpVisitor& operator=(const HelpVisitor& rhs);
+
+	protected:
+
+		/**
+		 * The CmdLine the output will be generated for. 
+		 */
+		CmdLineInterface* _cmd;
+
+		/**
+		 * The output object. 
+		 */
+		CmdLineOutput** _out;
+
+	public:
+
+		/**
+		 * Constructor.
+		 * \param cmd - The CmdLine the output will be generated for.
+		 * \param out - The type of output. 
+		 */
+		HelpVisitor(CmdLineInterface* cmd, CmdLineOutput** out) 
+				: Visitor(), _cmd( cmd ), _out( out ) { }
+
+		/**
+		 * Calls the usage method of the CmdLineOutput for the 
+		 * specified CmdLine.
+		 */
+		void visit() { (*_out)->usage(*_cmd); throw ExitException(0); }
+		
+};
+
+}
+
+#endif

runtime/onnxruntime/include/tclap/IgnoreRestVisitor.h

@@ -0,0 +1,54 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+
+/****************************************************************************** 
+ * 
+ *  file:  IgnoreRestVisitor.h
+ * 
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+
+#ifndef TCLAP_IGNORE_REST_VISITOR_H
+#define TCLAP_IGNORE_REST_VISITOR_H
+
+#include <tclap/Visitor.h>
+#include <tclap/Arg.h>
+
+namespace TCLAP {
+
+/**
+ * A Visitor that tells the CmdLine to begin ignoring arguments after
+ * this one is parsed.
+ */
+class IgnoreRestVisitor: public Visitor
+{
+	public:
+
+		/**
+		 * Constructor.
+		 */
+		IgnoreRestVisitor() : Visitor() {}
+
+		/**
+		 * Sets Arg::_ignoreRest.
+		 */
+		void visit() { Arg::beginIgnoring();  }
+};
+
+}
+
+#endif

runtime/onnxruntime/include/tclap/MultiArg.h

@@ -0,0 +1,433 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/****************************************************************************** 
+ * 
+ *  file:  MultiArg.h
+ * 
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  Copyright (c) 2004, Michael E. Smoot, Daniel Aarno.
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/
+
+
+#ifndef TCLAP_MULTIPLE_ARGUMENT_H
+#define TCLAP_MULTIPLE_ARGUMENT_H
+
+#include <string>
+#include <vector>
+
+#include <tclap/Arg.h>
+#include <tclap/Constraint.h>
+
+namespace TCLAP {
+/**
+ * An argument that allows multiple values of type T to be specified.  Very
+ * similar to a ValueArg, except a vector of values will be returned
+ * instead of just one.
+ */
+template<class T>
+class MultiArg : public Arg
+{
+public:
+	typedef std::vector<T> container_type;	
+	typedef typename container_type::iterator iterator;
+	typedef typename container_type::const_iterator const_iterator;
+
+protected:
+
+	/**
+	 * The list of values parsed from the CmdLine.
+	 */
+	std::vector<T> _values;
+
+	/**
+	 * The description of type T to be used in the usage.
+	 */
+	std::string _typeDesc;
+
+	/**
+	 * A list of constraint on this Arg. 
+	 */
+	Constraint<T>* _constraint;
+
+	/**
+	 * Extracts the value from the string.
+	 * Attempts to parse string as type T, if this fails an exception
+	 * is thrown.
+	 * \param val - The string to be read.
+	 */
+	void _extractValue( const std::string& val );
+
+	/**
+	 * Used by XorHandler to decide whether to keep parsing for this arg.
+	 */
+	bool _allowMore;
+
+public:
+
+	/**
+	 * Constructor.
+	 * \param flag - The one character flag that identifies this
+	 * argument on the command line.
+	 * \param name - A one word name for the argument.  Can be
+	 * used as a long flag on the command line.
+	 * \param desc - A description of what the argument is for or
+	 * does.
+	 * \param req - Whether the argument is required on the command
+	 * line.
+	 * \param typeDesc - A short, human readable description of the
+	 * type that this object expects.  This is used in the generation
+	 * of the USAGE statement.  The goal is to be helpful to the end user
+	 * of the program.
+	 * \param v - An optional visitor.  You probably should not
+	 * use this unless you have a very good reason.
+	 */
+	MultiArg( const std::string& flag,
+                  const std::string& name,
+                  const std::string& desc,
+                  bool req,
+                  const std::string& typeDesc,
+                  Visitor* v = NULL);
+
+	/**
+	 * Constructor.
+	 * \param flag - The one character flag that identifies this
+	 * argument on the command line.
+	 * \param name - A one word name for the argument.  Can be
+	 * used as a long flag on the command line.
+	 * \param desc - A description of what the argument is for or
+	 * does.
+	 * \param req - Whether the argument is required on the command
+	 * line.
+	 * \param typeDesc - A short, human readable description of the
+	 * type that this object expects.  This is used in the generation
+	 * of the USAGE statement.  The goal is to be helpful to the end user
+	 * of the program.
+	 * \param parser - A CmdLine parser object to add this Arg to
+	 * \param v - An optional visitor.  You probably should not
+	 * use this unless you have a very good reason.
+	 */
+	MultiArg( const std::string& flag, 
+                  const std::string& name,
+                  const std::string& desc,
+                  bool req,
+                  const std::string& typeDesc,
+                  CmdLineInterface& parser,
+                  Visitor* v = NULL );
+
+	/**
+	 * Constructor.
+	 * \param flag - The one character flag that identifies this
+	 * argument on the command line.
+	 * \param name - A one word name for the argument.  Can be
+	 * used as a long flag on the command line.
+	 * \param desc - A description of what the argument is for or
+	 * does.
+	 * \param req - Whether the argument is required on the command
+	 * line.
+	 * \param constraint - A pointer to a Constraint object used
+	 * to constrain this Arg.
+	 * \param v - An optional visitor.  You probably should not
+	 * use this unless you have a very good reason.
+	 */
+	MultiArg( const std::string& flag,
+                  const std::string& name,
+                  const std::string& desc,
+                  bool req,
+                  Constraint<T>* constraint,
+                  Visitor* v = NULL );
+		  
+	/**
+	 * Constructor.
+	 * \param flag - The one character flag that identifies this
+	 * argument on the command line.
+	 * \param name - A one word name for the argument.  Can be
+	 * used as a long flag on the command line.
+	 * \param desc - A description of what the argument is for or
+	 * does.
+	 * \param req - Whether the argument is required on the command
+	 * line.
+	 * \param constraint - A pointer to a Constraint object used
+	 * to constrain this Arg.
+	 * \param parser - A CmdLine parser object to add this Arg to
+	 * \param v - An optional visitor.  You probably should not
+	 * use this unless you have a very good reason.
+	 */
+	MultiArg( const std::string& flag, 
+                  const std::string& name,
+                  const std::string& desc,
+                  bool req,
+                  Constraint<T>* constraint,
+                  CmdLineInterface& parser,
+                  Visitor* v = NULL );
+		  
+	/**
+	 * Handles the processing of the argument.
+	 * This re-implements the Arg version of this method to set the
+	 * _value of the argument appropriately.  It knows the difference
+	 * between labeled and unlabeled.
+	 * \param i - Pointer the the current argument in the list.
+	 * \param args - Mutable list of strings. Passed from main().
+	 */
+	virtual bool processArg(int* i, std::vector<std::string>& args); 
+
+	/**
+	 * Returns a vector of type T containing the values parsed from
+	 * the command line.
+	 */
+	const std::vector<T>& getValue() const { return _values; }
+
+	/**
+	 * Returns an iterator over the values parsed from the command
+	 * line.
+	 */
+	const_iterator begin() const { return _values.begin(); }
+
+	/**
+	 * Returns the end of the values parsed from the command
+	 * line.
+	 */
+	const_iterator end() const { return _values.end(); }
+
+	/**
+	 * Returns the a short id string.  Used in the usage. 
+	 * \param val - value to be used.
+	 */
+	virtual std::string shortID(const std::string& val="val") const;
+
+	/**
+	 * Returns the a long id string.  Used in the usage. 
+	 * \param val - value to be used.
+	 */
+	virtual std::string longID(const std::string& val="val") const;
+
+	/**
+	 * Once we've matched the first value, then the arg is no longer
+	 * required.
+	 */
+	virtual bool isRequired() const;
+
+	virtual bool allowMore();
+	
+	virtual void reset();
+
+private:
+	/**
+	 * Prevent accidental copying
+	 */
+	MultiArg(const MultiArg<T>& rhs);
+	MultiArg& operator=(const MultiArg<T>& rhs);
+
+};
+
+template<class T>
+MultiArg<T>::MultiArg(const std::string& flag, 
+                      const std::string& name,
+                      const std::string& desc,
+                      bool req,
+                      const std::string& typeDesc,
+                      Visitor* v) :
+   Arg( flag, name, desc, req, true, v ),
+  _values(std::vector<T>()),
+  _typeDesc( typeDesc ),
+  _constraint( NULL ),
+  _allowMore(false)
+{ 
+	_acceptsMultipleValues = true;
+}
+
+template<class T>
+MultiArg<T>::MultiArg(const std::string& flag, 
+                      const std::string& name,
+                      const std::string& desc,
+                      bool req,
+                      const std::string& typeDesc,
+                      CmdLineInterface& parser,
+                      Visitor* v)
+: Arg( flag, name, desc, req, true, v ),
+  _values(std::vector<T>()),
+  _typeDesc( typeDesc ),
+  _constraint( NULL ),
+  _allowMore(false)
+{ 
+	parser.add( this );
+	_acceptsMultipleValues = true;
+}
+
+/**
+ *
+ */
+template<class T>
+MultiArg<T>::MultiArg(const std::string& flag, 
+                      const std::string& name,
+                      const std::string& desc,
+                      bool req,
+                      Constraint<T>* constraint,
+                      Visitor* v)
+: Arg( flag, name, desc, req, true, v ),
+  _values(std::vector<T>()),
+  _typeDesc( Constraint<T>::shortID(constraint) ),
+  _constraint( constraint ),
+  _allowMore(false)
+{ 
+	_acceptsMultipleValues = true;
+}
+
+template<class T>
+MultiArg<T>::MultiArg(const std::string& flag, 
+                      const std::string& name,
+                      const std::string& desc,
+                      bool req,
+                      Constraint<T>* constraint,
+                      CmdLineInterface& parser,
+                      Visitor* v)
+: Arg( flag, name, desc, req, true, v ),
+  _values(std::vector<T>()),
+  _typeDesc( Constraint<T>::shortID(constraint) ),
+  _constraint( constraint ),
+  _allowMore(false)
+{ 
+	parser.add( this );
+	_acceptsMultipleValues = true;
+}
+
+template<class T>
+bool MultiArg<T>::processArg(int *i, std::vector<std::string>& args) 
+{
+ 	if ( _ignoreable && Arg::ignoreRest() )
+		return false;
+
+	if ( _hasBlanks( args[*i] ) )
+		return false;
+
+	std::string flag = args[*i];
+	std::string value = "";
+
+   	trimFlag( flag, value );
+
+   	if ( argMatches( flag ) )
+   	{
+   		if ( Arg::delimiter() != ' ' && value == "" )
+			throw( ArgParseException( 
+			           "Couldn't find delimiter for this argument!",
+					   toString() ) );
+
+		// always take the first one, regardless of start string
+		if ( value == "" )
+		{
+			(*i)++;
+			if ( static_cast<unsigned int>(*i) < args.size() )
+				_extractValue( args[*i] );
+			else
+				throw( ArgParseException("Missing a value for this argument!",
+                                         toString() ) );
+		} 
+		else
+			_extractValue( value );
+
+		/*
+		// continuing taking the args until we hit one with a start string 
+		while ( (unsigned int)(*i)+1 < args.size() &&
+				args[(*i)+1].find_first_of( Arg::flagStartString() ) != 0 &&
+		        args[(*i)+1].find_first_of( Arg::nameStartString() ) != 0 ) 
+				_extractValue( args[++(*i)] );
+		*/
+
+		_alreadySet = true;
+		_checkWithVisitor();
+
+		return true;
+	}
+	else
+		return false;
+}
+
+/**
+ *
+ */
+template<class T>
+std::string MultiArg<T>::shortID(const std::string& val) const
+{
+	static_cast<void>(val); // Ignore input, don't warn
+	return Arg::shortID(_typeDesc) + " ...";
+}
+
+/**
+ *
+ */
+template<class T>
+std::string MultiArg<T>::longID(const std::string& val) const
+{
+	static_cast<void>(val); // Ignore input, don't warn
+	return Arg::longID(_typeDesc) + "  (accepted multiple times)";
+}
+
+/**
+ * Once we've matched the first value, then the arg is no longer
+ * required.
+ */
+template<class T>
+bool MultiArg<T>::isRequired() const
+{
+	if ( _required )
+	{
+		if ( _values.size() > 1 )
+			return false;
+		else
+			return true;
+   	}
+   	else
+		return false;
+
+}
+
+template<class T>
+void MultiArg<T>::_extractValue( const std::string& val ) 
+{
+    try {
+	T tmp;
+	ExtractValue(tmp, val, typename ArgTraits<T>::ValueCategory());
+	_values.push_back(tmp);
+    } catch( ArgParseException &e) {
+	throw ArgParseException(e.error(), toString());
+    }
+
+    if ( _constraint != NULL )
+	if ( ! _constraint->check( _values.back() ) )
+	    throw( CmdLineParseException( "Value '" + val +
+					  "' does not meet constraint: " +
+					  _constraint->description(), 
+					  toString() ) );
+}
+		
+template<class T>
+bool MultiArg<T>::allowMore()
+{
+	bool am = _allowMore;
+	_allowMore = true;
+	return am;
+}
+
+template<class T>
+void MultiArg<T>::reset()
+{
+	Arg::reset();
+	_values.clear();
+}
+
+} // namespace TCLAP
+
+#endif

runtime/onnxruntime/include/tclap/MultiSwitchArg.h

@@ -0,0 +1,217 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+
+/****************************************************************************** 
+*
+*  file:  MultiSwitchArg.h
+*
+*  Copyright (c) 2003, Michael E. Smoot .
+*  Copyright (c) 2004, Michael E. Smoot, Daniel Aarno.
+*  Copyright (c) 2005, Michael E. Smoot, Daniel Aarno, Erik Zeek.
+*  Copyright (c) 2017, Google LLC
+*  All rights reserved.
+*
+*  See the file COPYING in the top directory of this distribution for
+*  more information.
+*
+*  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS
+*  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+*  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*  DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************/
+
+
+#ifndef TCLAP_MULTI_SWITCH_ARG_H
+#define TCLAP_MULTI_SWITCH_ARG_H
+
+#include <string>
+#include <vector>
+
+#include <tclap/SwitchArg.h>
+
+namespace TCLAP {
+
+/**
+* A multiple switch argument.  If the switch is set on the command line, then
+* the getValue method will return the number of times the switch appears.
+*/
+class MultiSwitchArg : public SwitchArg
+{
+	protected:
+
+		/**
+		 * The value of the switch.
+		 */
+		int _value;
+
+		/**
+		 * Used to support the reset() method so that ValueArg can be
+		 * reset to their constructed value.
+		 */
+		int _default;
+
+	public:
+
+		/**
+		 * MultiSwitchArg constructor.
+		 * \param flag - The one character flag that identifies this
+		 * argument on the command line.
+		 * \param name - A one word name for the argument.  Can be
+		 * used as a long flag on the command line.
+		 * \param desc - A description of what the argument is for or
+		 * does.
+		 * \param init - Optional. The initial/default value of this Arg. 
+		 * Defaults to 0.
+		 * \param v - An optional visitor.  You probably should not
+		 * use this unless you have a very good reason.
+		 */
+		MultiSwitchArg(const std::string& flag, 
+				const std::string& name,
+				const std::string& desc,
+				int init = 0,
+				Visitor* v = NULL);
+
+
+		/**
+		 * MultiSwitchArg constructor.
+		 * \param flag - The one character flag that identifies this
+		 * argument on the command line.
+		 * \param name - A one word name for the argument.  Can be
+		 * used as a long flag on the command line.
+		 * \param desc - A description of what the argument is for or
+		 * does.
+		 * \param parser - A CmdLine parser object to add this Arg to
+		 * \param init - Optional. The initial/default value of this Arg. 
+		 * Defaults to 0.
+		 * \param v - An optional visitor.  You probably should not
+		 * use this unless you have a very good reason.
+		 */
+		MultiSwitchArg(const std::string& flag, 
+				const std::string& name,
+				const std::string& desc,
+				CmdLineInterface& parser,
+				int init = 0,
+				Visitor* v = NULL);
+
+
+		/**
+		 * Handles the processing of the argument.
+		 * This re-implements the SwitchArg version of this method to set the
+		 * _value of the argument appropriately.
+		 * \param i - Pointer the the current argument in the list.
+		 * \param args - Mutable list of strings. Passed
+		 * in from main().
+		 */
+		virtual bool processArg(int* i, std::vector<std::string>& args); 
+
+		/**
+		 * Returns int, the number of times the switch has been set.
+		 */
+		int getValue() const { return _value; }
+
+		/**
+		 * Returns the shortID for this Arg.
+		 */
+		std::string shortID(const std::string& val) const;
+
+		/**
+		 * Returns the longID for this Arg.
+		 */
+		std::string longID(const std::string& val) const;
+		
+		void reset();
+
+};
+
+//////////////////////////////////////////////////////////////////////
+//BEGIN MultiSwitchArg.cpp
+//////////////////////////////////////////////////////////////////////
+inline MultiSwitchArg::MultiSwitchArg(const std::string& flag,
+					const std::string& name,
+					const std::string& desc,
+					int init,
+					Visitor* v )
+: SwitchArg(flag, name, desc, false, v),
+_value( init ),
+_default( init )
+{ }
+
+inline MultiSwitchArg::MultiSwitchArg(const std::string& flag,
+					const std::string& name, 
+					const std::string& desc, 
+					CmdLineInterface& parser,
+					int init,
+					Visitor* v )
+: SwitchArg(flag, name, desc, false, v),
+_value( init ),
+_default( init )
+{ 
+	parser.add( this );
+}
+
+inline bool MultiSwitchArg::processArg(int *i, std::vector<std::string>& args)
+{
+	if ( _ignoreable && Arg::ignoreRest() )
+		return false;
+
+	if ( argMatches( args[*i] ))
+	{
+		// so the isSet() method will work
+		_alreadySet = true;
+
+		// Matched argument: increment value.
+		++_value;
+
+		_checkWithVisitor();
+
+		return true;
+	}
+	else if ( combinedSwitchesMatch( args[*i] ) )
+	{
+		// so the isSet() method will work
+		_alreadySet = true;
+
+		// Matched argument: increment value.
+		++_value;
+
+		// Check for more in argument and increment value.
+		while ( combinedSwitchesMatch( args[*i] ) ) 
+			++_value;
+
+		_checkWithVisitor();
+
+		return false;
+	}
+	else
+		return false;
+}
+
+inline std::string 
+MultiSwitchArg::shortID(const std::string& val) const
+{
+	return Arg::shortID(val) + " ...";
+}
+
+inline std::string 
+MultiSwitchArg::longID(const std::string& val) const
+{
+	return Arg::longID(val) + "  (accepted multiple times)";
+}
+
+inline void
+MultiSwitchArg::reset()
+{
+	MultiSwitchArg::_value = MultiSwitchArg::_default;
+}
+
+//////////////////////////////////////////////////////////////////////
+//END MultiSwitchArg.cpp
+//////////////////////////////////////////////////////////////////////
+
+} //namespace TCLAP
+
+#endif

runtime/onnxruntime/include/tclap/OptionalUnlabeledTracker.h

@@ -0,0 +1,64 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+
+
+/****************************************************************************** 
+ * 
+ *  file:  OptionalUnlabeledTracker.h
+ * 
+ *  Copyright (c) 2005, Michael E. Smoot .
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+
+#ifndef TCLAP_OPTIONAL_UNLABELED_TRACKER_H
+#define TCLAP_OPTIONAL_UNLABELED_TRACKER_H
+
+#include <string>
+
+namespace TCLAP {
+
+class OptionalUnlabeledTracker
+{
+
+	public:
+
+		static void check( bool req, const std::string& argName );
+
+		static void gotOptional() { alreadyOptionalRef() = true; }
+
+		static bool& alreadyOptional() { return alreadyOptionalRef(); } 
+
+	private:
+
+		static bool& alreadyOptionalRef() { static bool ct = false; return ct; }
+};
+
+
+inline void OptionalUnlabeledTracker::check( bool req, const std::string& argName )
+{
+    if ( OptionalUnlabeledTracker::alreadyOptional() )
+        throw( SpecificationException(
+	"You can't specify ANY Unlabeled Arg following an optional Unlabeled Arg",
+	                argName ) );
+
+    if ( !req )
+        OptionalUnlabeledTracker::gotOptional();
+}
+
+
+} // namespace TCLAP
+
+#endif

runtime/onnxruntime/include/tclap/StandardTraits.h

@@ -0,0 +1,63 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/******************************************************************************
+ *
+ *  file:  StandardTraits.h
+ *
+ *  Copyright (c) 2007, Daniel Aarno, Michael E. Smoot .
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ *
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ *  DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************/
+
+// This is an internal tclap file, you should probably not have to
+// include this directly
+
+#ifndef TCLAP_STANDARD_TRAITS_H
+#define TCLAP_STANDARD_TRAITS_H
+
+#ifdef HAVE_CONFIG_H
+#include <config.h> // To check for long long
+#endif
+
+// If Microsoft has already typedef'd wchar_t as an unsigned 
+// short, then compiles will break because it's as if we're
+// creating ArgTraits twice for unsigned short. Thus...
+#ifdef _MSC_VER
+#ifndef _NATIVE_WCHAR_T_DEFINED
+#define TCLAP_DONT_DECLARE_WCHAR_T_ARGTRAITS
+#endif
+#endif
+
+namespace TCLAP {
+
+// Integer types (signed, unsigned and bool) and floating point types all
+// have value-like semantics.
+
+// Strings have string like argument traits.
+template<>
+struct ArgTraits<std::string> {
+    typedef StringLike ValueCategory;
+};
+
+template<typename T>
+void SetString(T &dst, const std::string &src)
+{
+    dst = src;
+}
+
+} // namespace
+
+#endif
+

runtime/onnxruntime/include/tclap/StdOutput.h

@@ -0,0 +1,300 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/****************************************************************************** 
+ * 
+ *  file:  StdOutput.h
+ * 
+ *  Copyright (c) 2004, Michael E. Smoot
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+#ifndef TCLAP_STDCMDLINEOUTPUT_H
+#define TCLAP_STDCMDLINEOUTPUT_H
+
+#include <string>
+#include <vector>
+#include <list>
+#include <iostream>
+#include <algorithm>
+
+#include <tclap/CmdLineInterface.h>
+#include <tclap/CmdLineOutput.h>
+#include <tclap/XorHandler.h>
+#include <tclap/Arg.h>
+
+namespace TCLAP {
+
+/**
+ * A class that isolates any output from the CmdLine object so that it
+ * may be easily modified.
+ */
+class StdOutput : public CmdLineOutput
+{
+
+	public:
+
+		/**
+		 * Prints the usage to stdout.  Can be overridden to 
+		 * produce alternative behavior.
+		 * \param c - The CmdLine object the output is generated for. 
+		 */
+		virtual void usage(CmdLineInterface& c);
+
+		/**
+		 * Prints the version to stdout. Can be overridden 
+		 * to produce alternative behavior.
+		 * \param c - The CmdLine object the output is generated for. 
+		 */
+		virtual void version(CmdLineInterface& c);
+
+		/**
+		 * Prints (to stderr) an error message, short usage 
+		 * Can be overridden to produce alternative behavior.
+		 * \param c - The CmdLine object the output is generated for. 
+		 * \param e - The ArgException that caused the failure. 
+		 */
+		virtual void failure(CmdLineInterface& c, 
+				     ArgException& e );
+
+	protected:
+
+        /**
+         * Writes a brief usage message with short args.
+		 * \param c - The CmdLine object the output is generated for. 
+         * \param os - The stream to write the message to.
+         */
+        void _shortUsage( CmdLineInterface& c, std::ostream& os ) const;
+
+        /**
+		 * Writes a longer usage message with long and short args, 
+		 * provides descriptions and prints message.
+		 * \param c - The CmdLine object the output is generated for. 
+		 * \param os - The stream to write the message to.
+		 */
+		void _longUsage( CmdLineInterface& c, std::ostream& os ) const;
+
+		/**
+		 * This function inserts line breaks and indents long strings 
+		 * according the  params input. It will only break lines at spaces, 
+		 * commas and pipes.
+		 * \param os - The stream to be printed to.
+		 * \param s - The string to be printed.
+		 * \param maxWidth - The maxWidth allowed for the output line. 
+		 * \param indentSpaces - The number of spaces to indent the first line. 
+		 * \param secondLineOffset - The number of spaces to indent the second
+		 * and all subsequent lines in addition to indentSpaces.
+		 */
+		void spacePrint( std::ostream& os, 
+						 const std::string& s, 
+						 int maxWidth, 
+						 int indentSpaces, 
+						 int secondLineOffset ) const;
+
+};
+
+
+inline void StdOutput::version(CmdLineInterface& _cmd) 
+{
+	std::string progName = _cmd.getProgramName();
+	std::string xversion = _cmd.getVersion();
+
+	std::cout << std::endl << progName << "  version: " 
+			  << xversion << std::endl << std::endl;
+}
+
+inline void StdOutput::usage(CmdLineInterface& _cmd ) 
+{
+	std::cout << std::endl << "USAGE: " << std::endl << std::endl; 
+
+	_shortUsage( _cmd, std::cout );
+
+	std::cout << std::endl << std::endl << "Where: " << std::endl << std::endl;
+
+	_longUsage( _cmd, std::cout );
+
+	std::cout << std::endl; 
+
+}
+
+inline void StdOutput::failure( CmdLineInterface& _cmd,
+								ArgException& e ) 
+{
+	std::string progName = _cmd.getProgramName();
+
+	std::cerr << "PARSE ERROR: " << e.argId() << std::endl
+		      << "             " << e.error() << std::endl << std::endl;
+
+	if ( _cmd.hasHelpAndVersion() )
+		{
+			std::cerr << "Brief USAGE: " << std::endl;
+
+			_shortUsage( _cmd, std::cerr );	
+
+			std::cerr << std::endl << "For complete USAGE and HELP type: " 
+					  << std::endl << "   " << progName << " "
+					  << Arg::nameStartString() << "help"
+					  << std::endl << std::endl;
+		}
+	else
+		usage(_cmd);
+
+	throw ExitException(1);
+}
+
+inline void 
+StdOutput::_shortUsage( CmdLineInterface& _cmd, 
+						std::ostream& os ) const
+{
+	std::list<Arg*> argList = _cmd.getArgList();
+	std::string progName = _cmd.getProgramName();
+	XorHandler xorHandler = _cmd.getXorHandler();
+	std::vector< std::vector<Arg*> > xorList = xorHandler.getXorList();
+
+	std::string s = progName + " ";
+
+	// first the xor
+	for ( int i = 0; static_cast<unsigned int>(i) < xorList.size(); i++ )
+		{
+			s += " {";
+			for ( ArgVectorIterator it = xorList[i].begin(); 
+				  it != xorList[i].end(); it++ )
+				s += (*it)->shortID() + "|";
+
+			s[s.length()-1] = '}';
+		}
+
+	// then the rest
+	for (ArgListIterator it = argList.begin(); it != argList.end(); it++)
+		if ( !xorHandler.contains( (*it) ) )
+			s += " " + (*it)->shortID();
+
+	// if the program name is too long, then adjust the second line offset 
+	int secondLineOffset = static_cast<int>(progName.length()) + 2;
+	if ( secondLineOffset > 75/2 )
+		secondLineOffset = static_cast<int>(75/2);
+
+	spacePrint( os, s, 75, 3, secondLineOffset );
+}
+
+inline void 
+StdOutput::_longUsage( CmdLineInterface& _cmd, 
+					   std::ostream& os ) const
+{
+	std::list<Arg*> argList = _cmd.getArgList();
+	std::string message = _cmd.getMessage();
+	XorHandler xorHandler = _cmd.getXorHandler();
+	std::vector< std::vector<Arg*> > xorList = xorHandler.getXorList();
+
+	// first the xor 
+	for ( int i = 0; static_cast<unsigned int>(i) < xorList.size(); i++ )
+		{
+			for ( ArgVectorIterator it = xorList[i].begin(); 
+				  it != xorList[i].end(); 
+				  it++ )
+				{
+					spacePrint( os, (*it)->longID(), 75, 3, 3 );
+					spacePrint( os, (*it)->getDescription(), 75, 5, 0 );
+
+					if ( it+1 != xorList[i].end() )
+						spacePrint(os, "-- OR --", 75, 9, 0);
+				}
+			os << std::endl << std::endl;
+		}
+
+	// then the rest
+	for (ArgListIterator it = argList.begin(); it != argList.end(); it++)
+		if ( !xorHandler.contains( (*it) ) )
+			{
+				spacePrint( os, (*it)->longID(), 75, 3, 3 ); 
+				spacePrint( os, (*it)->getDescription(), 75, 5, 0 ); 
+				os << std::endl;
+			}
+
+	os << std::endl;
+
+	spacePrint( os, message, 75, 3, 0 );
+}
+
+inline void StdOutput::spacePrint( std::ostream& os, 
+						           const std::string& s, 
+						           int maxWidth, 
+						           int indentSpaces, 
+						           int secondLineOffset ) const
+{
+	int len = static_cast<int>(s.length());
+
+	if ( (len + indentSpaces > maxWidth) && maxWidth > 0 )
+		{
+			int allowedLen = maxWidth - indentSpaces;
+			int start = 0;
+			while ( start < len )
+				{
+					// find the substring length
+					// int stringLen = std::min<int>( len - start, allowedLen );
+					// doing it this way to support a VisualC++ 2005 bug 
+					using namespace std; 
+					int stringLen = min<int>( len - start, allowedLen );
+
+					// trim the length so it doesn't end in middle of a word
+					if ( stringLen == allowedLen )
+						while ( stringLen >= 0 &&
+								s[stringLen+start] != ' ' && 
+								s[stringLen+start] != ',' &&
+								s[stringLen+start] != '|' ) 
+							stringLen--;
+	
+					// ok, the word is longer than the line, so just split 
+					// wherever the line ends
+					if ( stringLen <= 0 )
+						stringLen = allowedLen;
+
+					// check for newlines
+					for ( int i = 0; i < stringLen; i++ )
+						if ( s[start+i] == '\n' )
+							stringLen = i+1;
+
+					// print the indent	
+					for ( int i = 0; i < indentSpaces; i++ )
+						os << " ";
+
+					if ( start == 0 )
+						{
+							// handle second line offsets
+							indentSpaces += secondLineOffset;
+
+							// adjust allowed len
+							allowedLen -= secondLineOffset;
+						}
+
+					os << s.substr(start,stringLen) << std::endl;
+
+					// so we don't start a line with a space
+					while ( s[stringLen+start] == ' ' && start < len )
+						start++;
+			
+					start += stringLen;
+				}
+		}
+	else
+		{
+			for ( int i = 0; i < indentSpaces; i++ )
+				os << " ";
+			os << s << std::endl;
+		}
+}
+
+} //namespace TCLAP
+#endif 

runtime/onnxruntime/include/tclap/SwitchArg.h

@@ -0,0 +1,273 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/****************************************************************************** 
+ * 
+ *  file:  SwitchArg.h
+ * 
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  Copyright (c) 2004, Michael E. Smoot, Daniel Aarno.
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/
+
+
+#ifndef TCLAP_SWITCH_ARG_H
+#define TCLAP_SWITCH_ARG_H
+
+#include <string>
+#include <vector>
+
+#include <tclap/Arg.h>
+
+namespace TCLAP {
+
+/**
+ * A simple switch argument.  If the switch is set on the command line, then
+ * the getValue method will return the opposite of the default value for the
+ * switch.
+ */
+class SwitchArg : public Arg
+{
+protected:
+
+  /**
+   * The value of the switch.
+   */
+  bool _value;
+
+  /**
+   * Used to support the reset() method so that ValueArg can be
+   * reset to their constructed value.
+   */
+  bool _default;
+
+public:
+
+  /**
+   * SwitchArg constructor.
+   * \param flag - The one character flag that identifies this
+   * argument on the command line.
+   * \param name - A one word name for the argument.  Can be
+   * used as a long flag on the command line.
+   * \param desc - A description of what the argument is for or
+   * does.
+   * \param def - The default value for this Switch. 
+   * \param v - An optional visitor.  You probably should not
+   * use this unless you have a very good reason.
+   */
+  SwitchArg(const std::string& flag, 
+	    const std::string& name, 
+	    const std::string& desc,
+	    bool def = false,
+	    Visitor* v = NULL);
+
+                                  
+  /**
+   * SwitchArg constructor.
+   * \param flag - The one character flag that identifies this
+   * argument on the command line.
+   * \param name - A one word name for the argument.  Can be
+   * used as a long flag on the command line.
+   * \param desc - A description of what the argument is for or
+   * does.
+   * \param parser - A CmdLine parser object to add this Arg to
+   * \param def - The default value for this Switch.
+   * \param v - An optional visitor.  You probably should not
+   * use this unless you have a very good reason.
+   */
+  SwitchArg(const std::string& flag, 
+	    const std::string& name, 
+	    const std::string& desc,
+	    CmdLineInterface& parser,
+	    bool def = false,
+	    Visitor* v = NULL);
+                                  
+                                  
+  /**
+   * Handles the processing of the argument.
+   * This re-implements the Arg version of this method to set the
+   * _value of the argument appropriately.
+   * \param i - Pointer the the current argument in the list.
+   * \param args - Mutable list of strings. Passed
+   * in from main().
+   */
+  virtual bool processArg(int* i, std::vector<std::string>& args); 
+
+  /**
+   * Checks a string to see if any of the chars in the string
+   * match the flag for this Switch.
+   */
+  bool combinedSwitchesMatch(std::string& combined);
+
+  /**
+   * Returns bool, whether or not the switch has been set.
+   */
+  bool getValue() const { return _value; }
+
+  /**
+   * A SwitchArg can be used as a boolean, indicating
+   * whether or not the switch has been set. This is the
+   * same as calling getValue()
+   */
+  operator bool() const { return _value; }
+                
+  virtual void reset();
+
+private:
+  /**
+   * Checks to see if we've found the last match in
+   * a combined string.
+   */
+  bool lastCombined(std::string& combined);
+
+  /**
+   * Does the common processing of processArg.
+   */
+  void commonProcessing();
+};
+
+//////////////////////////////////////////////////////////////////////
+//BEGIN SwitchArg.cpp
+//////////////////////////////////////////////////////////////////////
+inline SwitchArg::SwitchArg(const std::string& flag, 
+			    const std::string& name, 
+			    const std::string& desc, 
+			    bool default_val,
+			    Visitor* v )
+  : Arg(flag, name, desc, false, false, v),
+    _value( default_val ),
+    _default( default_val )
+{ }
+
+inline SwitchArg::SwitchArg(const std::string& flag, 
+			    const std::string& name, 
+			    const std::string& desc, 
+			    CmdLineInterface& parser,
+			    bool default_val,
+			    Visitor* v )
+  : Arg(flag, name, desc, false, false, v),
+    _value( default_val ),
+    _default(default_val)
+{ 
+  parser.add( this );
+}
+
+inline bool SwitchArg::lastCombined(std::string& combinedSwitches ) 
+{
+  for ( unsigned int i = 1; i < combinedSwitches.length(); i++ )
+    if ( combinedSwitches[i] != Arg::blankChar() )
+      return false;
+        
+  return true;
+}
+
+inline bool SwitchArg::combinedSwitchesMatch(std::string& combinedSwitches )
+{
+  // make sure this is actually a combined switch
+  if ( combinedSwitches.length() > 0 &&
+       combinedSwitches[0] != Arg::flagStartString()[0] )
+    return false;
+
+  // make sure it isn't a long name 
+  if ( combinedSwitches.substr( 0, Arg::nameStartString().length() ) == 
+       Arg::nameStartString() )
+    return false;
+
+  // make sure the delimiter isn't in the string 
+  if ( combinedSwitches.find_first_of(Arg::delimiter()) != std::string::npos)
+    return false;
+
+  // ok, we're not specifying a ValueArg, so we know that we have
+  // a combined switch list.  
+  for ( unsigned int i = 1; i < combinedSwitches.length(); i++ )
+    if ( _flag.length() > 0 && 
+	 combinedSwitches[i] == _flag[0] &&
+	 _flag[0] != Arg::flagStartString()[0] ) 
+      {
+	// update the combined switches so this one is no longer present
+	// this is necessary so that no unlabeled args are matched
+	// later in the processing.
+	//combinedSwitches.erase(i,1);
+	combinedSwitches[i] = Arg::blankChar(); 
+	return true;
+      }
+
+  // none of the switches passed in the list match. 
+  return false;   
+}
+
+inline void SwitchArg::commonProcessing()
+{
+  if ( _xorSet )
+    throw(CmdLineParseException(
+				"Mutually exclusive argument already set!", toString()));
+
+  if ( _alreadySet ) 
+    throw(CmdLineParseException("Argument already set!", toString()));
+
+  _alreadySet = true;
+
+  if ( _value == true )
+    _value = false;
+  else
+    _value = true;
+
+  _checkWithVisitor();
+}
+
+inline bool SwitchArg::processArg(int *i, std::vector<std::string>& args)
+{
+  if ( _ignoreable && Arg::ignoreRest() )
+    return false;
+
+  // if the whole string matches the flag or name string
+  if ( argMatches( args[*i] ) )
+    {
+      commonProcessing();
+
+      return true;
+    }
+  // if a substring matches the flag as part of a combination
+  else if ( combinedSwitchesMatch( args[*i] ) )
+    {
+      // check again to ensure we don't misinterpret 
+      // this as a MultiSwitchArg 
+      if ( combinedSwitchesMatch( args[*i] ) )
+	throw(CmdLineParseException("Argument already set!", 
+				    toString()));
+
+      commonProcessing();
+
+      // We only want to return true if we've found the last combined
+      // match in the string, otherwise we return true so that other 
+      // switches in the combination will have a chance to match.
+      return lastCombined( args[*i] );
+    }
+  else
+    return false;
+}
+
+inline void SwitchArg::reset()
+{
+  Arg::reset();
+  _value = _default;  
+}
+//////////////////////////////////////////////////////////////////////
+//End SwitchArg.cpp
+//////////////////////////////////////////////////////////////////////
+
+} //namespace TCLAP
+
+#endif

runtime/onnxruntime/include/tclap/UnlabeledMultiArg.h

@@ -0,0 +1,304 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+
+/****************************************************************************** 
+ * 
+ *  file:  UnlabeledMultiArg.h
+ * 
+ *  Copyright (c) 2003, Michael E. Smoot.
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+
+#ifndef TCLAP_MULTIPLE_UNLABELED_ARGUMENT_H
+#define TCLAP_MULTIPLE_UNLABELED_ARGUMENT_H
+
+#include <string>
+#include <vector>
+
+#include <tclap/MultiArg.h>
+#include <tclap/OptionalUnlabeledTracker.h>
+
+namespace TCLAP {
+
+/**
+ * Just like a MultiArg, except that the arguments are unlabeled.  Basically,
+ * this Arg will slurp up everything that hasn't been matched to another 
+ * Arg.
+ */
+template<class T>
+class UnlabeledMultiArg : public MultiArg<T>
+{
+
+	// If compiler has two stage name lookup (as gcc >= 3.4 does)
+	// this is required to prevent undef. symbols
+	using MultiArg<T>::_ignoreable;
+	using MultiArg<T>::_hasBlanks;
+	using MultiArg<T>::_extractValue;
+	using MultiArg<T>::_typeDesc;
+	using MultiArg<T>::_name;
+	using MultiArg<T>::_description;
+	using MultiArg<T>::_alreadySet;
+	using MultiArg<T>::toString;
+
+	public:
+		
+		/**
+		 * Constructor.  
+		 * \param name - The name of the Arg. Note that this is used for
+		 * identification, not as a long flag.
+		 * \param desc - A description of what the argument is for or
+		 * does.
+		 * \param req - Whether the argument is required on the command
+		 *  line.
+		 * \param typeDesc - A short, human readable description of the
+		 * type that this object expects.  This is used in the generation
+		 * of the USAGE statement.  The goal is to be helpful to the end user
+		 * of the program.
+		 * \param ignoreable - Whether or not this argument can be ignored
+		 * using the "--" flag.
+		 * \param v - An optional visitor.  You probably should not
+		 * use this unless you have a very good reason.
+		 */
+		UnlabeledMultiArg( const std::string& name,
+				           const std::string& desc,
+						   bool req,
+				           const std::string& typeDesc,
+						   bool ignoreable = false,
+				           Visitor* v = NULL );
+		/**
+		 * Constructor.  
+		 * \param name - The name of the Arg. Note that this is used for
+		 * identification, not as a long flag.
+		 * \param desc - A description of what the argument is for or
+		 * does.
+		 * \param req - Whether the argument is required on the command
+		 *  line.
+		 * \param typeDesc - A short, human readable description of the
+		 * type that this object expects.  This is used in the generation
+		 * of the USAGE statement.  The goal is to be helpful to the end user
+		 * of the program.
+		 * \param parser - A CmdLine parser object to add this Arg to
+		 * \param ignoreable - Whether or not this argument can be ignored
+		 * using the "--" flag.
+		 * \param v - An optional visitor.  You probably should not
+		 * use this unless you have a very good reason.
+		 */
+		UnlabeledMultiArg( const std::string& name,
+				           const std::string& desc,
+						   bool req,
+				           const std::string& typeDesc,
+						   CmdLineInterface& parser,
+						   bool ignoreable = false,
+				           Visitor* v = NULL );
+						 
+		/**
+		 * Constructor.  
+		 * \param name - The name of the Arg. Note that this is used for
+		 * identification, not as a long flag.
+		 * \param desc - A description of what the argument is for or
+		 * does.
+		 * \param req - Whether the argument is required on the command
+		 *  line.
+		 * \param constraint - A pointer to a Constraint object used
+		 * to constrain this Arg.
+		 * \param ignoreable - Whether or not this argument can be ignored
+		 * using the "--" flag.
+		 * \param v - An optional visitor.  You probably should not
+		 * use this unless you have a very good reason.
+		 */
+		UnlabeledMultiArg( const std::string& name,
+						   const std::string& desc,
+						   bool req,
+						   Constraint<T>* constraint,
+						   bool ignoreable = false,
+						   Visitor* v = NULL );
+
+		/**
+		 * Constructor.  
+		 * \param name - The name of the Arg. Note that this is used for
+		 * identification, not as a long flag.
+		 * \param desc - A description of what the argument is for or
+		 * does.
+		 * \param req - Whether the argument is required on the command
+		 *  line.
+		 * \param constraint - A pointer to a Constraint object used
+		 * to constrain this Arg.
+		 * \param parser - A CmdLine parser object to add this Arg to
+		 * \param ignoreable - Whether or not this argument can be ignored
+		 * using the "--" flag.
+		 * \param v - An optional visitor.  You probably should not
+		 * use this unless you have a very good reason.
+		 */
+		UnlabeledMultiArg( const std::string& name, 
+						   const std::string& desc, 
+						   bool req,
+						   Constraint<T>* constraint,
+						   CmdLineInterface& parser,
+						   bool ignoreable = false,
+						   Visitor* v = NULL );
+						 
+		/**
+		 * Handles the processing of the argument.
+		 * This re-implements the Arg version of this method to set the
+		 * _value of the argument appropriately.  It knows the difference
+		 * between labeled and unlabeled.
+		 * \param i - Pointer the the current argument in the list.
+		 * \param args - Mutable list of strings. Passed from main().
+		 */
+		virtual bool processArg(int* i, std::vector<std::string>& args); 
+
+		/**
+		 * Returns the a short id string.  Used in the usage.
+		 * \param val - value to be used.
+		 */
+		virtual std::string shortID(const std::string& val="val") const;
+
+		/**
+		 * Returns the a long id string.  Used in the usage.
+		 * \param val - value to be used.
+		 */
+		virtual std::string longID(const std::string& val="val") const;
+
+		/**
+		 * Operator ==.
+		 * \param a - The Arg to be compared to this.
+		 */
+		virtual bool operator==(const Arg& a) const;
+
+		/**
+		 * Pushes this to back of list rather than front.
+		 * \param argList - The list this should be added to.
+		 */
+		virtual void addToList( std::list<Arg*>& argList ) const;
+};
+
+template<class T>
+UnlabeledMultiArg<T>::UnlabeledMultiArg(const std::string& name, 
+				                        const std::string& desc, 
+										bool req,
+					                    const std::string& typeDesc,
+										bool ignoreable,
+					                    Visitor* v)
+: MultiArg<T>("", name, desc,  req, typeDesc, v)
+{ 
+	_ignoreable = ignoreable;
+	OptionalUnlabeledTracker::check(true, toString());
+}
+
+template<class T>
+UnlabeledMultiArg<T>::UnlabeledMultiArg(const std::string& name, 
+				                        const std::string& desc, 
+										bool req,
+					                    const std::string& typeDesc,
+										CmdLineInterface& parser,
+										bool ignoreable,
+					                    Visitor* v)
+: MultiArg<T>("", name, desc,  req, typeDesc, v)
+{ 
+	_ignoreable = ignoreable;
+	OptionalUnlabeledTracker::check(true, toString());
+	parser.add( this );
+}
+
+
+template<class T>
+UnlabeledMultiArg<T>::UnlabeledMultiArg(const std::string& name, 
+				                        const std::string& desc, 
+										bool req,
+					                    Constraint<T>* constraint,
+										bool ignoreable,
+					                    Visitor* v)
+: MultiArg<T>("", name, desc,  req, constraint, v)
+{ 
+	_ignoreable = ignoreable;
+	OptionalUnlabeledTracker::check(true, toString());
+}
+
+template<class T>
+UnlabeledMultiArg<T>::UnlabeledMultiArg(const std::string& name, 
+				                        const std::string& desc, 
+										bool req,
+					                    Constraint<T>* constraint,
+										CmdLineInterface& parser,
+										bool ignoreable,
+					                    Visitor* v)
+: MultiArg<T>("", name, desc,  req, constraint, v)
+{ 
+	_ignoreable = ignoreable;
+	OptionalUnlabeledTracker::check(true, toString());
+	parser.add( this );
+}
+
+
+template<class T>
+bool UnlabeledMultiArg<T>::processArg(int *i, std::vector<std::string>& args) 
+{
+
+	if ( _hasBlanks( args[*i] ) )
+		return false;
+
+	// never ignore an unlabeled multi arg
+
+
+	// always take the first value, regardless of the start string 
+	_extractValue( args[(*i)] );
+
+	/*
+	// continue taking args until we hit the end or a start string 
+	while ( (unsigned int)(*i)+1 < args.size() &&
+			args[(*i)+1].find_first_of( Arg::flagStartString() ) != 0 &&
+            args[(*i)+1].find_first_of( Arg::nameStartString() ) != 0 ) 
+		_extractValue( args[++(*i)] );
+	*/
+
+	_alreadySet = true;
+
+	return true;
+}
+
+template<class T>
+std::string UnlabeledMultiArg<T>::shortID(const std::string& val) const
+{
+	static_cast<void>(val); // Ignore input, don't warn
+	return std::string("<") + _typeDesc + "> ...";
+}
+
+template<class T>
+std::string UnlabeledMultiArg<T>::longID(const std::string& val) const
+{
+	static_cast<void>(val); // Ignore input, don't warn
+	return std::string("<") + _typeDesc + ">  (accepted multiple times)";
+}
+
+template<class T>
+bool UnlabeledMultiArg<T>::operator==(const Arg& a) const
+{
+	if ( _name == a.getName() || _description == a.getDescription() )
+		return true;
+	else
+		return false;
+}
+
+template<class T>
+void UnlabeledMultiArg<T>::addToList( std::list<Arg*>& argList ) const
+{
+	argList.push_back( const_cast<Arg*>(static_cast<const Arg* const>(this)) );
+}
+
+}
+
+#endif

runtime/onnxruntime/include/tclap/UnlabeledValueArg.h

@@ -0,0 +1,343 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+
+/****************************************************************************** 
+ * 
+ *  file:  UnlabeledValueArg.h
+ * 
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  Copyright (c) 2004, Michael E. Smoot, Daniel Aarno.
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+
+#ifndef TCLAP_UNLABELED_VALUE_ARGUMENT_H
+#define TCLAP_UNLABELED_VALUE_ARGUMENT_H
+
+#include <string>
+#include <vector>
+
+#include <tclap/ValueArg.h>
+#include <tclap/OptionalUnlabeledTracker.h>
+
+
+namespace TCLAP {
+
+/**
+ * The basic unlabeled argument that parses a value.
+ * This is a template class, which means the type T defines the type
+ * that a given object will attempt to parse when an UnlabeledValueArg
+ * is reached in the list of args that the CmdLine iterates over.
+ */
+template<class T>
+class UnlabeledValueArg : public ValueArg<T>
+{
+
+	// If compiler has two stage name lookup (as gcc >= 3.4 does)
+	// this is required to prevent undef. symbols
+	using ValueArg<T>::_ignoreable;
+	using ValueArg<T>::_hasBlanks;
+	using ValueArg<T>::_extractValue;
+	using ValueArg<T>::_typeDesc;
+	using ValueArg<T>::_name;
+	using ValueArg<T>::_description;
+	using ValueArg<T>::_alreadySet;
+	using ValueArg<T>::toString;
+
+	public:
+
+		/**
+		 * UnlabeledValueArg constructor.
+		 * \param name - A one word name for the argument.  Note that this is used for
+		 * identification, not as a long flag.
+		 * \param desc - A description of what the argument is for or
+		 * does.
+		 * \param req - Whether the argument is required on the command
+		 * line.
+		 * \param value - The default value assigned to this argument if it
+		 * is not present on the command line.
+		 * \param typeDesc - A short, human readable description of the
+		 * type that this object expects.  This is used in the generation
+		 * of the USAGE statement.  The goal is to be helpful to the end user
+		 * of the program.
+		 * \param ignoreable - Allows you to specify that this argument can be
+		 * ignored if the '--' flag is set.  This defaults to false (cannot
+		 * be ignored) and should  generally stay that way unless you have 
+		 * some special need for certain arguments to be ignored.
+		 * \param v - Optional Visitor.  You should leave this blank unless
+		 * you have a very good reason.
+		 */
+		UnlabeledValueArg( const std::string& name, 
+			               const std::string& desc, 
+						   bool req,
+				           T value,
+				           const std::string& typeDesc,
+						   bool ignoreable = false,
+				           Visitor* v = NULL); 
+
+		/**
+		 * UnlabeledValueArg constructor.
+		 * \param name - A one word name for the argument.  Note that this is used for
+		 * identification, not as a long flag.
+		 * \param desc - A description of what the argument is for or
+		 * does.
+		 * \param req - Whether the argument is required on the command
+		 * line.
+		 * \param value - The default value assigned to this argument if it
+		 * is not present on the command line.
+		 * \param typeDesc - A short, human readable description of the
+		 * type that this object expects.  This is used in the generation
+		 * of the USAGE statement.  The goal is to be helpful to the end user
+		 * of the program.
+		 * \param parser - A CmdLine parser object to add this Arg to
+		 * \param ignoreable - Allows you to specify that this argument can be
+		 * ignored if the '--' flag is set.  This defaults to false (cannot
+		 * be ignored) and should  generally stay that way unless you have 
+		 * some special need for certain arguments to be ignored.
+		 * \param v - Optional Visitor.  You should leave this blank unless
+		 * you have a very good reason.
+		 */
+		UnlabeledValueArg( const std::string& name, 
+			               const std::string& desc, 
+						   bool req,
+				           T value,
+				           const std::string& typeDesc,
+						   CmdLineInterface& parser,
+						   bool ignoreable = false,
+				           Visitor* v = NULL ); 					
+						
+		/**
+		 * UnlabeledValueArg constructor.
+		 * \param name - A one word name for the argument.  Note that this is used for
+		 * identification, not as a long flag.
+		 * \param desc - A description of what the argument is for or
+		 * does.
+		 * \param req - Whether the argument is required on the command
+		 * line.
+		 * \param value - The default value assigned to this argument if it
+		 * is not present on the command line.
+		 * \param constraint - A pointer to a Constraint object used
+		 * to constrain this Arg.
+		 * \param ignoreable - Allows you to specify that this argument can be
+		 * ignored if the '--' flag is set.  This defaults to false (cannot
+		 * be ignored) and should  generally stay that way unless you have 
+		 * some special need for certain arguments to be ignored.
+		 * \param v - Optional Visitor.  You should leave this blank unless
+		 * you have a very good reason.
+		 */
+		UnlabeledValueArg( const std::string& name, 
+			               const std::string& desc, 
+						   bool req,
+				           T value,
+				           Constraint<T>* constraint,
+						   bool ignoreable = false,
+				           Visitor* v = NULL ); 
+
+		
+		/**
+		 * UnlabeledValueArg constructor.
+		 * \param name - A one word name for the argument.  Note that this is used for
+		 * identification, not as a long flag.
+		 * \param desc - A description of what the argument is for or
+		 * does.
+		 * \param req - Whether the argument is required on the command
+		 * line.
+		 * \param value - The default value assigned to this argument if it
+		 * is not present on the command line.
+		 * \param constraint - A pointer to a Constraint object used
+		 * to constrain this Arg.
+		 * \param parser - A CmdLine parser object to add this Arg to
+		 * \param ignoreable - Allows you to specify that this argument can be
+		 * ignored if the '--' flag is set.  This defaults to false (cannot
+		 * be ignored) and should  generally stay that way unless you have 
+		 * some special need for certain arguments to be ignored.
+		 * \param v - Optional Visitor.  You should leave this blank unless
+		 * you have a very good reason.
+		 */
+		UnlabeledValueArg( const std::string& name, 
+			               const std::string& desc, 
+						   bool req,
+				           T value,
+				           Constraint<T>* constraint,
+						   CmdLineInterface& parser,
+						   bool ignoreable = false,
+				           Visitor* v = NULL);
+						
+		/**
+		 * Handles the processing of the argument.
+		 * This re-implements the Arg version of this method to set the
+		 * _value of the argument appropriately.  Handling specific to
+		 * unlabeled arguments.
+		 * \param i - Pointer the the current argument in the list.
+		 * \param args - Mutable list of strings. 
+		 */
+		virtual bool processArg(int* i, std::vector<std::string>& args); 
+
+		/**
+		 * Overrides shortID for specific behavior.
+		 */
+		virtual std::string shortID(const std::string& val="val") const;
+
+		/**
+		 * Overrides longID for specific behavior.
+		 */
+		virtual std::string longID(const std::string& val="val") const;
+
+		/**
+		 * Overrides operator== for specific behavior.
+		 */
+		virtual bool operator==(const Arg& a ) const;
+
+		/**
+		 * Instead of pushing to the front of list, push to the back.
+		 * \param argList - The list to add this to.
+		 */
+		virtual void addToList( std::list<Arg*>& argList ) const;
+
+};
+
+/**
+ * Constructor implementation.
+ */
+template<class T>
+UnlabeledValueArg<T>::UnlabeledValueArg(const std::string& name, 
+					                    const std::string& desc, 
+										bool req,
+					                    T val,
+					                    const std::string& typeDesc,
+					                    bool ignoreable,
+					                    Visitor* v)
+: ValueArg<T>("", name, desc, req, val, typeDesc, v)
+{ 
+	_ignoreable = ignoreable;
+
+	OptionalUnlabeledTracker::check(req, toString());
+
+}
+
+template<class T>
+UnlabeledValueArg<T>::UnlabeledValueArg(const std::string& name, 
+					                    const std::string& desc, 
+										bool req,
+					                    T val,
+					                    const std::string& typeDesc,
+					                    CmdLineInterface& parser,
+					                    bool ignoreable,
+					                    Visitor* v)
+: ValueArg<T>("", name, desc, req, val, typeDesc, v)
+{ 
+	_ignoreable = ignoreable;
+	OptionalUnlabeledTracker::check(req, toString());
+	parser.add( this );
+}
+
+/**
+ * Constructor implementation.
+ */
+template<class T>
+UnlabeledValueArg<T>::UnlabeledValueArg(const std::string& name, 
+                                        const std::string& desc, 
+										bool req,
+                                        T val,
+                                        Constraint<T>* constraint,
+                                        bool ignoreable,
+                                        Visitor* v)
+: ValueArg<T>("", name, desc, req, val, constraint, v)
+{ 
+	_ignoreable = ignoreable;
+	OptionalUnlabeledTracker::check(req, toString());
+}
+
+template<class T>
+UnlabeledValueArg<T>::UnlabeledValueArg(const std::string& name, 
+					                    const std::string& desc, 
+										bool req,
+					                    T val,
+					                    Constraint<T>* constraint,
+					                    CmdLineInterface& parser,
+					                    bool ignoreable,
+					                    Visitor* v)
+: ValueArg<T>("", name, desc, req, val, constraint,  v)
+{ 
+	_ignoreable = ignoreable;
+	OptionalUnlabeledTracker::check(req, toString());
+	parser.add( this );
+}
+
+/**
+ * Implementation of processArg().
+ */
+template<class T>
+bool UnlabeledValueArg<T>::processArg(int *i, std::vector<std::string>& args) 
+{
+	
+	if ( _alreadySet )
+		return false;
+	
+	if ( _hasBlanks( args[*i] ) )
+		return false;
+
+	// never ignore an unlabeled arg
+	
+	_extractValue( args[*i] );
+	_alreadySet = true;
+	return true;
+}
+
+/**
+ * Overriding shortID for specific output.
+ */
+template<class T>
+std::string UnlabeledValueArg<T>::shortID(const std::string& val) const
+{
+	static_cast<void>(val); // Ignore input, don't warn
+	return std::string("<") + _typeDesc + ">";
+}
+
+/**
+ * Overriding longID for specific output.
+ */
+template<class T>
+std::string UnlabeledValueArg<T>::longID(const std::string& val) const
+{
+	static_cast<void>(val); // Ignore input, don't warn
+
+	// Ideally we would like to be able to use RTTI to return the name
+	// of the type required for this argument.  However, g++ at least, 
+	// doesn't appear to return terribly useful "names" of the types.  
+	return std::string("<") + _typeDesc + ">";
+}
+
+/**
+ * Overriding operator== for specific behavior.
+ */
+template<class T>
+bool UnlabeledValueArg<T>::operator==(const Arg& a ) const
+{
+	if ( _name == a.getName() || _description == a.getDescription() )
+		return true;
+	else
+		return false;
+}
+
+template<class T>
+void UnlabeledValueArg<T>::addToList( std::list<Arg*>& argList ) const
+{
+	argList.push_back( const_cast<Arg*>(static_cast<const Arg* const>(this)) );
+}
+
+}
+#endif

runtime/onnxruntime/include/tclap/ValueArg.h

@@ -0,0 +1,430 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/****************************************************************************** 
+ * 
+ *  file:  ValueArg.h
+ * 
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  Copyright (c) 2004, Michael E. Smoot, Daniel Aarno.
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/
+
+
+#ifndef TCLAP_VALUE_ARGUMENT_H
+#define TCLAP_VALUE_ARGUMENT_H
+
+#include <string>
+#include <vector>
+
+#include <tclap/Arg.h>
+#include <tclap/Constraint.h>
+
+namespace TCLAP {
+
+/**
+ * The basic labeled argument that parses a value.
+ * This is a template class, which means the type T defines the type
+ * that a given object will attempt to parse when the flag/name is matched
+ * on the command line.  While there is nothing stopping you from creating
+ * an unflagged ValueArg, it is unwise and would cause significant problems.
+ * Instead use an UnlabeledValueArg.
+ */
+template<class T>
+class ValueArg : public Arg 
+{
+protected:
+
+  /**
+   * The value parsed from the command line.
+   * Can be of any type, as long as the >> operator for the type
+   * is defined.
+   */
+  T _value;
+
+  /**
+   * Used to support the reset() method so that ValueArg can be
+   * reset to their constructed value.
+   */
+  T _default;
+
+  /**
+   * A human readable description of the type to be parsed.
+   * This is a hack, plain and simple.  Ideally we would use RTTI to
+   * return the name of type T, but until there is some sort of
+   * consistent support for human readable names, we are left to our
+   * own devices.
+   */
+  std::string _typeDesc;
+
+  /**
+   * A Constraint this Arg must conform to. 
+   */
+  Constraint<T>* _constraint;
+
+  /**
+   * Extracts the value from the string.
+   * Attempts to parse string as type T, if this fails an exception
+   * is thrown.
+   * \param val - value to be parsed. 
+   */
+  void _extractValue( const std::string& val );
+
+public:
+
+  /**
+   * Labeled ValueArg constructor.
+   * You could conceivably call this constructor with a blank flag, 
+   * but that would make you a bad person.  It would also cause
+   * an exception to be thrown.   If you want an unlabeled argument, 
+   * use the other constructor.
+   * \param flag - The one character flag that identifies this
+   * argument on the command line.
+   * \param name - A one word name for the argument.  Can be
+   * used as a long flag on the command line.
+   * \param desc - A description of what the argument is for or
+   * does.
+   * \param req - Whether the argument is required on the command
+   * line.
+   * \param value - The default value assigned to this argument if it
+   * is not present on the command line.
+   * \param typeDesc - A short, human readable description of the
+   * type that this object expects.  This is used in the generation
+   * of the USAGE statement.  The goal is to be helpful to the end user
+   * of the program.
+   * \param v - An optional visitor.  You probably should not
+   * use this unless you have a very good reason.
+   */
+  ValueArg( const std::string& flag, 
+            const std::string& name, 
+            const std::string& desc, 
+            bool req, 
+            T value,
+            const std::string& typeDesc,
+            Visitor* v = NULL);
+                                 
+                                 
+  /**
+   * Labeled ValueArg constructor.
+   * You could conceivably call this constructor with a blank flag, 
+   * but that would make you a bad person.  It would also cause
+   * an exception to be thrown.   If you want an unlabeled argument, 
+   * use the other constructor.
+   * \param flag - The one character flag that identifies this
+   * argument on the command line.
+   * \param name - A one word name for the argument.  Can be
+   * used as a long flag on the command line.
+   * \param desc - A description of what the argument is for or
+   * does.
+   * \param req - Whether the argument is required on the command
+   * line.
+   * \param value - The default value assigned to this argument if it
+   * is not present on the command line.
+   * \param typeDesc - A short, human readable description of the
+   * type that this object expects.  This is used in the generation
+   * of the USAGE statement.  The goal is to be helpful to the end user
+   * of the program.
+   * \param parser - A CmdLine parser object to add this Arg to
+   * \param v - An optional visitor.  You probably should not
+   * use this unless you have a very good reason.
+   */
+  ValueArg( const std::string& flag, 
+            const std::string& name, 
+            const std::string& desc, 
+            bool req, 
+            T value,
+            const std::string& typeDesc,
+            CmdLineInterface& parser,
+            Visitor* v = NULL );
+ 
+  /**
+   * Labeled ValueArg constructor.
+   * You could conceivably call this constructor with a blank flag, 
+   * but that would make you a bad person.  It would also cause
+   * an exception to be thrown.   If you want an unlabeled argument, 
+   * use the other constructor.
+   * \param flag - The one character flag that identifies this
+   * argument on the command line.
+   * \param name - A one word name for the argument.  Can be
+   * used as a long flag on the command line.
+   * \param desc - A description of what the argument is for or
+   * does.
+   * \param req - Whether the argument is required on the command
+   * line.
+   * \param value - The default value assigned to this argument if it
+   * is not present on the command line.
+   * \param constraint - A pointer to a Constraint object used
+   * to constrain this Arg.
+   * \param parser - A CmdLine parser object to add this Arg to.
+   * \param v - An optional visitor.  You probably should not
+   * use this unless you have a very good reason.
+   */
+  ValueArg( const std::string& flag, 
+            const std::string& name, 
+            const std::string& desc, 
+            bool req, 
+            T value,
+            Constraint<T>* constraint,
+            CmdLineInterface& parser,
+            Visitor* v = NULL );
+          
+  /**
+   * Labeled ValueArg constructor.
+   * You could conceivably call this constructor with a blank flag, 
+   * but that would make you a bad person.  It would also cause
+   * an exception to be thrown.   If you want an unlabeled argument, 
+   * use the other constructor.
+   * \param flag - The one character flag that identifies this
+   * argument on the command line.
+   * \param name - A one word name for the argument.  Can be
+   * used as a long flag on the command line.
+   * \param desc - A description of what the argument is for or
+   * does.
+   * \param req - Whether the argument is required on the command
+   * line.
+   * \param value - The default value assigned to this argument if it
+   * is not present on the command line.
+   * \param constraint - A pointer to a Constraint object used
+   * to constrain this Arg.
+   * \param v - An optional visitor.  You probably should not
+   * use this unless you have a very good reason.
+   */
+  ValueArg( const std::string& flag, 
+            const std::string& name, 
+            const std::string& desc, 
+            bool req, 
+            T value,
+            Constraint<T>* constraint,
+            Visitor* v = NULL );
+
+  /**
+   * Handles the processing of the argument.
+   * This re-implements the Arg version of this method to set the
+   * _value of the argument appropriately.  It knows the difference
+   * between labeled and unlabeled.
+   * \param i - Pointer the the current argument in the list.
+   * \param args - Mutable list of strings. Passed 
+   * in from main().
+   */
+  virtual bool processArg(int* i, std::vector<std::string>& args); 
+
+  /**
+   * Returns the value of the argument.
+   */
+  const T& getValue() const { return _value; }
+
+  // TODO(macbishop): Non-const variant is deprecated, don't
+  // use. Remove in next major.
+  T& getValue() { return _value; }
+
+  /**
+   * A ValueArg can be used as as its value type (T) This is the
+   * same as calling getValue()
+   */
+  operator const T&() const { return getValue(); }
+
+  /**
+   * Specialization of shortID.
+   * \param val - value to be used.
+   */
+  virtual std::string shortID(const std::string& val = "val") const;
+
+  /**
+   * Specialization of longID.
+   * \param val - value to be used.
+   */
+  virtual std::string longID(const std::string& val = "val") const;
+        
+  virtual void reset() ;
+
+private:
+  /**
+   * Prevent accidental copying
+   */
+  ValueArg(const ValueArg<T>& rhs);
+  ValueArg& operator=(const ValueArg<T>& rhs);
+};
+
+
+/**
+ * Constructor implementation.
+ */
+template<class T>
+ValueArg<T>::ValueArg(const std::string& flag, 
+                      const std::string& name, 
+                      const std::string& desc, 
+                      bool req, 
+                      T val,
+                      const std::string& typeDesc,
+                      Visitor* v)
+  : Arg(flag, name, desc, req, true, v),
+    _value( val ),
+    _default( val ),
+    _typeDesc( typeDesc ),
+    _constraint( NULL )
+{ }
+
+template<class T>
+ValueArg<T>::ValueArg(const std::string& flag, 
+                      const std::string& name, 
+                      const std::string& desc, 
+                      bool req, 
+                      T val,
+                      const std::string& typeDesc,
+                      CmdLineInterface& parser,
+                      Visitor* v)
+  : Arg(flag, name, desc, req, true, v),
+    _value( val ),
+    _default( val ),
+    _typeDesc( typeDesc ),
+    _constraint( NULL )
+{ 
+  parser.add( this );
+}
+
+template<class T>
+ValueArg<T>::ValueArg(const std::string& flag, 
+                      const std::string& name, 
+                      const std::string& desc, 
+                      bool req, 
+                      T val,
+                      Constraint<T>* constraint,
+                      Visitor* v)
+  : Arg(flag, name, desc, req, true, v),
+    _value( val ),
+    _default( val ),
+    _typeDesc( Constraint<T>::shortID(constraint) ),
+    _constraint( constraint )
+{ }
+
+template<class T>
+ValueArg<T>::ValueArg(const std::string& flag, 
+                      const std::string& name, 
+                      const std::string& desc, 
+                      bool req, 
+                      T val,
+                      Constraint<T>* constraint,
+                      CmdLineInterface& parser,
+                      Visitor* v)
+  : Arg(flag, name, desc, req, true, v),
+    _value( val ),
+    _default( val ),
+    _typeDesc( Constraint<T>::shortID(constraint) ),  // TODO(macbishop): Will crash
+    // if constraint is NULL
+    _constraint( constraint )
+{ 
+  parser.add( this );
+}
+
+/**
+ * Implementation of processArg().
+ */
+template<class T>
+bool ValueArg<T>::processArg(int *i, std::vector<std::string>& args)
+{
+  if ( _ignoreable && Arg::ignoreRest() )
+    return false;
+
+  if ( _hasBlanks( args[*i] ) )
+    return false;
+
+  std::string flag = args[*i];
+
+  std::string value = "";
+  trimFlag( flag, value );
+
+  if ( argMatches( flag ) )
+    {
+      if ( _alreadySet )
+        {
+          if ( _xorSet )
+            throw( CmdLineParseException("Mutually exclusive argument"
+                                         " already set!", toString()));
+          else
+            throw( CmdLineParseException("Argument already set!", 
+                                         toString()) );
+        }
+
+      if ( Arg::delimiter() != ' ' && value == "" )
+        throw( ArgParseException("Couldn't find delimiter for this argument!",
+                                 toString() ) );
+
+      if ( value == "" )
+        {
+          (*i)++;
+          if ( static_cast<unsigned int>(*i) < args.size() ) 
+            _extractValue( args[*i] );
+          else
+            throw( ArgParseException("Missing a value for this argument!",
+                                     toString() ) );
+        }
+      else
+        _extractValue( value );
+                                
+      _alreadySet = true;
+      _checkWithVisitor();
+      return true;
+    }   
+  else
+    return false;
+}
+
+/**
+ * Implementation of shortID.
+ */
+template<class T>
+std::string ValueArg<T>::shortID(const std::string& val) const
+{
+  static_cast<void>(val); // Ignore input, don't warn
+  return Arg::shortID( _typeDesc ); 
+}
+
+/**
+ * Implementation of longID.
+ */
+template<class T>
+std::string ValueArg<T>::longID(const std::string& val) const
+{
+  static_cast<void>(val); // Ignore input, don't warn
+  return Arg::longID( _typeDesc ); 
+}
+
+template<class T>
+void ValueArg<T>::_extractValue( const std::string& val ) 
+{
+  try {
+    ExtractValue(_value, val, typename ArgTraits<T>::ValueCategory());
+  } catch( ArgParseException &e) {
+    throw ArgParseException(e.error(), toString());
+  }
+    
+  if ( _constraint != NULL )
+    if ( ! _constraint->check( _value ) )
+      throw( CmdLineParseException( "Value '" + val + 
+                                    + "' does not meet constraint: " 
+                                    + _constraint->description(),
+                                    toString() ) );
+}
+
+template<class T>
+void ValueArg<T>::reset()
+{
+  Arg::reset();
+  _value = _default;
+}
+
+} // namespace TCLAP
+
+#endif

runtime/onnxruntime/include/tclap/ValuesConstraint.h

@@ -0,0 +1,134 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+
+
+/****************************************************************************** 
+ * 
+ *  file:  ValuesConstraint.h
+ * 
+ *  Copyright (c) 2005, Michael E. Smoot
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+#ifndef TCLAP_VALUESCONSTRAINT_H
+#define TCLAP_VALUESCONSTRAINT_H
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <string>
+#include <vector>
+#include <tclap/Constraint.h>
+#include <tclap/sstream.h>
+
+namespace TCLAP {
+
+/**
+ * A Constraint that constrains the Arg to only those values specified
+ * in the constraint.
+ */
+template<class T>
+class ValuesConstraint : public Constraint<T>
+{
+
+	public:
+
+		/**
+		 * Constructor. 
+		 * \param allowed - vector of allowed values. 
+		 */
+		ValuesConstraint(std::vector<T>const& allowed);
+
+		/**
+		 * Virtual destructor.
+		 */
+		virtual ~ValuesConstraint() {}
+
+		/**
+		 * Returns a description of the Constraint. 
+		 */
+		virtual std::string description() const;
+
+		/**
+		 * Returns the short ID for the Constraint.
+		 */
+		virtual std::string shortID() const;
+
+		/**
+		 * The method used to verify that the value parsed from the command
+		 * line meets the constraint.
+		 * \param value - The value that will be checked. 
+		 */
+		virtual bool check(const T& value) const;
+	
+	protected:
+
+		/**
+		 * The list of valid values. 
+		 */
+		std::vector<T> _allowed;
+
+		/**
+		 * The string used to describe the allowed values of this constraint.
+		 */
+		std::string _typeDesc;
+
+};
+
+template<class T>
+ValuesConstraint<T>::ValuesConstraint(std::vector<T> const& allowed)
+: _allowed(allowed),
+  _typeDesc("")
+{ 
+    for ( unsigned int i = 0; i < _allowed.size(); i++ )
+    {
+        std::ostringstream os;
+        os << _allowed[i];
+
+        std::string temp( os.str() ); 
+
+        if ( i > 0 )
+			_typeDesc += "|";
+        _typeDesc += temp;
+    }
+}
+
+template<class T>
+bool ValuesConstraint<T>::check( const T& val ) const
+{
+	if ( std::find(_allowed.begin(),_allowed.end(),val) == _allowed.end() )
+		return false;
+	else 
+		return true;
+}
+
+template<class T>
+std::string ValuesConstraint<T>::shortID() const
+{
+    return _typeDesc;	
+}
+
+template<class T>
+std::string ValuesConstraint<T>::description() const
+{
+    return _typeDesc;	
+}
+
+
+} //namespace TCLAP
+#endif 
+

runtime/onnxruntime/include/tclap/VersionVisitor.h

@@ -0,0 +1,81 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/****************************************************************************** 
+ * 
+ *  file:  VersionVisitor.h
+ * 
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+
+#ifndef TCLAP_VERSION_VISITOR_H
+#define TCLAP_VERSION_VISITOR_H
+
+#include <tclap/CmdLineInterface.h>
+#include <tclap/CmdLineOutput.h>
+#include <tclap/Visitor.h>
+
+namespace TCLAP {
+
+/**
+ * A Visitor that will call the version method of the given CmdLineOutput
+ * for the specified CmdLine object and then exit.
+ */
+class VersionVisitor: public Visitor
+{
+	private:
+		/**
+		 * Prevent accidental copying
+		 */
+		VersionVisitor(const VersionVisitor& rhs);
+		VersionVisitor& operator=(const VersionVisitor& rhs);
+
+	protected:
+
+		/**
+		 * The CmdLine of interest.
+		 */
+		CmdLineInterface* _cmd;
+
+		/**
+		 * The output object. 
+		 */
+		CmdLineOutput** _out;
+
+	public:
+
+		/**
+		 * Constructor.
+		 * \param cmd - The CmdLine the output is generated for. 
+		 * \param out - The type of output. 
+		 */
+		VersionVisitor( CmdLineInterface* cmd, CmdLineOutput** out ) 
+				: Visitor(), _cmd( cmd ), _out( out ) { }
+
+		/**
+		 * Calls the version method of the output object using the
+		 * specified CmdLine.
+		 */
+		void visit() { 
+		    (*_out)->version(*_cmd); 
+		    throw ExitException(0); 
+		}
+
+};
+
+}
+
+#endif

runtime/onnxruntime/include/tclap/Visitor.h

@@ -0,0 +1,57 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+
+/****************************************************************************** 
+ * 
+ *  file:  Visitor.h
+ * 
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  Copyright (c) 2017, Google LLC
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+
+#ifndef TCLAP_VISITOR_H
+#define TCLAP_VISITOR_H
+
+namespace TCLAP {
+
+/**
+ * A base class that defines the interface for visitors.
+ */
+class Visitor
+{
+	public:
+
+		/**
+		 * Constructor. Does nothing.
+		 */
+		Visitor() { }
+
+		/**
+		 * Destructor. Does nothing.
+		 */
+		virtual ~Visitor() { }
+
+		/**
+		 * This method (to implemented by children) will be
+		 * called when the visitor is visited.
+		 */
+		virtual void visit() = 0;
+};
+
+}
+
+#endif

runtime/onnxruntime/include/tclap/XorHandler.h

@@ -0,0 +1,168 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+
+/****************************************************************************** 
+ * 
+ *  file:  XorHandler.h
+ * 
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  Copyright (c) 2004, Michael E. Smoot, Daniel Aarno.
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.  
+ *  
+ *****************************************************************************/ 
+
+#ifndef TCLAP_XORHANDLER_H
+#define TCLAP_XORHANDLER_H
+
+#include <tclap/Arg.h>
+#include <string>
+#include <vector>
+#include <algorithm>
+#include <iostream>
+
+namespace TCLAP {
+
+/**
+ * This class handles lists of Arg's that are to be XOR'd on the command
+ * line.  This is used by CmdLine and you shouldn't ever use it.
+ */
+class XorHandler
+{
+	protected:
+
+		/**
+		 * The list of of lists of Arg's to be or'd together.
+		 */
+		std::vector< std::vector<Arg*> > _orList;
+
+	public:
+
+		/**
+		 * Constructor.  Does nothing.
+		 */
+		XorHandler( ) : _orList(std::vector< std::vector<Arg*> >()) {}
+
+		/**
+		 * Add a list of Arg*'s that will be xor'd together.
+		 * \param ors - list of Arg* that will be xor'd.
+		 */
+		void add( const std::vector<Arg*>& ors );
+
+		/**
+		 * Checks whether the specified Arg is in one of the xor lists and
+		 * if it does match one, returns the size of the xor list that the
+		 * Arg matched.  If the Arg matches, then it also sets the rest of
+		 * the Arg's in the list. You shouldn't use this.  
+		 * \param a - The Arg to be checked.
+		 */
+		int check( const Arg* a );
+
+		/**
+		 * Returns the XOR specific short usage.
+		 */
+		std::string shortUsage();
+
+		/**
+		 * Prints the XOR specific long usage.
+		 * \param os - Stream to print to.
+		 */
+		void printLongUsage(std::ostream& os);
+
+		/**
+		 * Simply checks whether the Arg is contained in one of the arg
+		 * lists.
+		 * \param a - The Arg to be checked.
+		 */
+		bool contains( const Arg* a );
+
+		const std::vector< std::vector<Arg*> >& getXorList() const;
+
+};
+
+
+//////////////////////////////////////////////////////////////////////
+//BEGIN XOR.cpp
+//////////////////////////////////////////////////////////////////////
+inline void XorHandler::add( const std::vector<Arg*>& ors )
+{
+	_orList.push_back( ors );
+}
+
+inline int XorHandler::check( const Arg* a ) 
+{
+	// iterate over each XOR list
+	for ( int i = 0; static_cast<unsigned int>(i) < _orList.size(); i++ )
+	{
+		// if the XOR list contains the arg..
+		ArgVectorIterator ait = std::find( _orList[i].begin(), 
+		                                   _orList[i].end(), a );
+		if ( ait != _orList[i].end() )
+		{
+			// first check to see if a mutually exclusive switch
+			// has not already been set
+			for ( ArgVectorIterator it = _orList[i].begin(); 
+				  it != _orList[i].end(); 
+				  it++ )
+				if ( a != (*it) && (*it)->isSet() )
+					throw(CmdLineParseException(
+					      "Mutually exclusive argument already set!",
+					      (*it)->toString()));
+
+			// go through and set each arg that is not a
+			for ( ArgVectorIterator it = _orList[i].begin(); 
+				  it != _orList[i].end(); 
+				  it++ )
+				if ( a != (*it) )
+					(*it)->xorSet();
+
+			// return the number of required args that have now been set
+			if ( (*ait)->allowMore() )
+				return 0;
+			else
+				return static_cast<int>(_orList[i].size());
+		}
+	}
+
+	if ( a->isRequired() )
+		return 1;
+	else
+		return 0;
+}
+
+inline bool XorHandler::contains( const Arg* a )
+{
+	for ( int i = 0; static_cast<unsigned int>(i) < _orList.size(); i++ )
+		for ( ArgVectorIterator it = _orList[i].begin(); 
+			  it != _orList[i].end(); 
+			  it++ )	
+			if ( a == (*it) )
+				return true;
+
+	return false;
+}
+
+inline const std::vector< std::vector<Arg*> >& XorHandler::getXorList() const
+{
+	return _orList;
+}
+
+
+
+//////////////////////////////////////////////////////////////////////
+//END XOR.cpp
+//////////////////////////////////////////////////////////////////////
+
+} //namespace TCLAP
+
+#endif 

runtime/onnxruntime/include/tclap/ZshCompletionOutput.h

@@ -0,0 +1,336 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/****************************************************************************** 
+ * 
+ *  file:  ZshCompletionOutput.h
+ * 
+ *  Copyright (c) 2006, Oliver Kiddle
+ *  Copyright (c) 2017 Google Inc.
+ *  All rights reserved.
+ * 
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *  
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+ *  DEALINGS IN THE SOFTWARE.
+ *  
+ *****************************************************************************/ 
+
+#ifndef TCLAP_ZSHCOMPLETIONOUTPUT_H
+#define TCLAP_ZSHCOMPLETIONOUTPUT_H
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <string>
+#include <vector>
+#include <list>
+#include <iostream>
+#include <map>
+
+#include <tclap/CmdLineInterface.h>
+#include <tclap/CmdLineOutput.h>
+#include <tclap/XorHandler.h>
+#include <tclap/Arg.h>
+#include <tclap/sstream.h>
+
+namespace TCLAP {
+
+/**
+ * A class that generates a Zsh completion function as output from the usage()
+ * method for the given CmdLine and its Args.
+ */
+class ZshCompletionOutput : public CmdLineOutput
+{
+
+	public:
+
+		ZshCompletionOutput();
+
+		/**
+		 * Prints the usage to stdout.  Can be overridden to 
+		 * produce alternative behavior.
+		 * \param c - The CmdLine object the output is generated for. 
+		 */
+		virtual void usage(CmdLineInterface& c);
+
+		/**
+		 * Prints the version to stdout. Can be overridden 
+		 * to produce alternative behavior.
+		 * \param c - The CmdLine object the output is generated for. 
+		 */
+		virtual void version(CmdLineInterface& c);
+
+		/**
+		 * Prints (to stderr) an error message, short usage 
+		 * Can be overridden to produce alternative behavior.
+		 * \param c - The CmdLine object the output is generated for. 
+		 * \param e - The ArgException that caused the failure. 
+		 */
+		virtual void failure(CmdLineInterface& c,
+						     ArgException& e );
+
+	protected:
+
+		void basename( std::string& s );
+		void quoteSpecialChars( std::string& s );
+
+		std::string getMutexList( CmdLineInterface& _cmd, Arg* a );
+		void printOption( Arg* it, std::string mutex );
+		void printArg( Arg* it );
+
+		std::map<std::string, std::string> common;
+		char theDelimiter;
+};
+
+ZshCompletionOutput::ZshCompletionOutput()
+: common(std::map<std::string, std::string>()),
+  theDelimiter('=')
+{
+	common["host"] = "_hosts";
+	common["hostname"] = "_hosts";
+	common["file"] = "_files";
+	common["filename"] = "_files";
+	common["user"] = "_users";
+	common["username"] = "_users";
+	common["directory"] = "_directories";
+	common["path"] = "_directories";
+	common["url"] = "_urls";
+}
+
+inline void ZshCompletionOutput::version(CmdLineInterface& _cmd)
+{
+	std::cout << _cmd.getVersion() << std::endl;
+}
+
+inline void ZshCompletionOutput::usage(CmdLineInterface& _cmd )
+{
+	std::list<Arg*> argList = _cmd.getArgList();
+	std::string progName = _cmd.getProgramName();
+	std::string xversion = _cmd.getVersion();
+	theDelimiter = _cmd.getDelimiter();
+	basename(progName);
+
+	std::cout << "#compdef " << progName << std::endl << std::endl <<
+		"# " << progName << " version " << _cmd.getVersion() << std::endl << std::endl <<
+		"_arguments -s -S";
+
+	for (ArgListIterator it = argList.begin(); it != argList.end(); it++)
+	{
+		if ( (*it)->shortID().at(0) == '<' )
+			printArg((*it));
+		else if ( (*it)->getFlag() != "-" )
+			printOption((*it), getMutexList(_cmd, *it));
+	}
+
+	std::cout << std::endl;
+}
+
+inline void ZshCompletionOutput::failure( CmdLineInterface& _cmd,
+				                ArgException& e )
+{
+	static_cast<void>(_cmd); // unused
+	std::cout << e.what() << std::endl;
+}
+
+inline void ZshCompletionOutput::quoteSpecialChars( std::string& s )
+{
+	size_t idx = s.find_last_of(':');
+	while ( idx != std::string::npos )
+	{
+		s.insert(idx, 1, '\\');
+		idx = s.find_last_of(':', idx);
+	}
+	idx = s.find_last_of('\'');
+	while ( idx != std::string::npos )
+	{
+		s.insert(idx, "'\\'");
+		if (idx == 0)
+			idx = std::string::npos;
+		else
+			idx = s.find_last_of('\'', --idx);
+	}
+}
+
+inline void ZshCompletionOutput::basename( std::string& s )
+{
+	size_t p = s.find_last_of('/');
+	if ( p != std::string::npos )
+	{
+		s.erase(0, p + 1);
+	}
+}
+
+inline void ZshCompletionOutput::printArg(Arg* a)
+{
+	static int count = 1;
+
+	std::cout << " \\" << std::endl << "  '";
+	if ( a->acceptsMultipleValues() )
+		std::cout << '*';
+	else
+		std::cout << count++;
+	std::cout << ':';
+	if ( !a->isRequired() )
+		std::cout << ':';
+
+	std::cout << a->getName() << ':';
+	std::map<std::string, std::string>::iterator compArg = common.find(a->getName());
+	if ( compArg != common.end() )
+	{
+		std::cout << compArg->second;
+	}
+	else
+	{
+		std::cout << "_guard \"^-*\" " << a->getName();
+	}
+	std::cout << '\'';
+}
+
+inline void ZshCompletionOutput::printOption(Arg* a, std::string mutex)
+{
+	std::string flag = a->flagStartChar() + a->getFlag();
+	std::string name = a->nameStartString() + a->getName();
+	std::string desc = a->getDescription();
+
+	// remove full stop and capitalization from description as
+	// this is the convention for zsh function
+	if (!desc.compare(0, 12, "(required)  "))
+	{
+		desc.erase(0, 12);
+	}
+	if (!desc.compare(0, 15, "(OR required)  "))
+	{
+		desc.erase(0, 15);
+	}
+	size_t len = desc.length();
+	if (len && desc.at(--len) == '.')
+	{
+		desc.erase(len);
+	}
+	if (len)
+	{
+		desc.replace(0, 1, 1, tolower(desc.at(0)));
+	}
+
+	std::cout << " \\" << std::endl << "  '" << mutex;
+
+	if ( a->getFlag().empty() )
+	{
+		std::cout << name;
+	}
+	else
+	{
+		std::cout << "'{" << flag << ',' << name << "}'";
+	}
+	if ( theDelimiter == '=' && a->isValueRequired() )
+		std::cout << "=-";
+	quoteSpecialChars(desc);
+	std::cout << '[' << desc << ']';
+
+	if ( a->isValueRequired() )
+	{
+		std::string arg = a->shortID();
+        // Example arg: "[-A <integer>] ..."
+        size_t pos = arg.rfind(" ...");
+
+        if (pos != std::string::npos) {
+            arg.erase(pos);
+        }
+
+		arg.erase(0, arg.find_last_of(theDelimiter) + 1);
+		if ( arg.at(arg.length()-1) == ']' )
+			arg.erase(arg.length()-1);
+		if ( arg.at(arg.length()-1) == ']' )
+		{
+			arg.erase(arg.length()-1);
+		}
+		if ( arg.at(0) == '<' )
+		{
+			arg.erase(arg.length()-1);
+			arg.erase(0, 1);
+		}
+		size_t p = arg.find('|');
+		if ( p != std::string::npos )
+		{
+			do
+			{
+				arg.replace(p, 1, 1, ' ');
+			}
+			while ( (p = arg.find_first_of('|', p)) != std::string::npos );
+			quoteSpecialChars(arg);
+			std::cout << ": :(" << arg << ')';
+		}
+		else
+		{
+			std::cout << ':' << arg;
+			std::map<std::string, std::string>::iterator compArg = common.find(arg);
+			if ( compArg != common.end() )
+			{
+				std::cout << ':' << compArg->second;
+			}
+		}
+	}
+
+	std::cout << '\'';
+}
+
+inline std::string ZshCompletionOutput::getMutexList( CmdLineInterface& _cmd, Arg* a)
+{
+	XorHandler xorHandler = _cmd.getXorHandler();
+	std::vector< std::vector<Arg*> > xorList = xorHandler.getXorList();
+	
+	if (a->getName() == "help" || a->getName() == "version")
+	{
+		return "(-)";
+	}
+
+	ostringstream list;
+	if ( a->acceptsMultipleValues() )
+	{
+		list << '*';
+	}
+
+	for ( int i = 0; static_cast<unsigned int>(i) < xorList.size(); i++ )
+	{
+		for ( ArgVectorIterator it = xorList[i].begin();
+			it != xorList[i].end();
+			it++)
+		if ( a == (*it) )
+		{
+			list << '(';
+			for ( ArgVectorIterator iu = xorList[i].begin();
+				iu != xorList[i].end();
+				iu++ )
+			{
+				bool notCur = (*iu) != a;
+				bool hasFlag = !(*iu)->getFlag().empty();
+				if ( iu != xorList[i].begin() && (notCur || hasFlag) )
+					list << ' ';
+				if (hasFlag)
+					list << (*iu)->flagStartChar() << (*iu)->getFlag() << ' ';
+				if ( notCur || hasFlag )
+					list << (*iu)->nameStartString() << (*iu)->getName();
+			}
+			list << ')';
+			return list.str();
+		}
+	}
+	
+	// wasn't found in xor list
+	if (!a->getFlag().empty()) {
+		list << "(" << a->flagStartChar() << a->getFlag() << ' ' <<
+			a->nameStartString() << a->getName() << ')';
+	}
+	
+	return list.str();
+}
+
+} //namespace TCLAP
+#endif

runtime/onnxruntime/include/tclap/sstream.h

@@ -0,0 +1,50 @@
+// -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*-
+
+/******************************************************************************
+ *
+ *  file:  sstream.h
+ *
+ *  Copyright (c) 2003, Michael E. Smoot .
+ *  Copyright (c) 2004, Michael E. Smoot, Daniel Aarno .
+ *  Copyright (c) 2017 Google Inc.
+ *  All rights reserved.
+ *
+ *  See the file COPYING in the top directory of this distribution for
+ *  more information.
+ *
+ *  THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ *  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ *  DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************/
+
+#ifndef TCLAP_SSTREAM_H
+#define TCLAP_SSTREAM_H
+
+#if !defined(HAVE_STRSTREAM)
+// Assume sstream is available if strstream is not specified
+// (https://sourceforge.net/p/tclap/bugs/23/)
+#define HAVE_SSTREAM
+#endif
+
+#if defined(HAVE_SSTREAM)
+#include <sstream>
+namespace TCLAP {
+    typedef std::istringstream istringstream;
+    typedef std::ostringstream ostringstream;
+}
+#elif defined(HAVE_STRSTREAM)
+#include <strstream>
+namespace TCLAP {
+    typedef std::istrstream istringstream;
+    typedef std::ostrstream ostringstream;
+}
+#else
+#error "Need a stringstream (sstream or strstream) to compile!"
+#endif
+
+#endif  // TCLAP_SSTREAM_H

runtime/onnxruntime/include/tpass-online-stream.h

@@ -0,0 +1,20 @@
+#ifndef TPASS_ONLINE_STREAM_H
+#define TPASS_ONLINE_STREAM_H
+
+#include <memory>
+#include "tpass-stream.h"
+#include "model.h"
+#include "vad-model.h"
+
+namespace funasr {
+class TpassOnlineStream {
+  public:
+    TpassOnlineStream(TpassStream* tpass_stream, std::vector<int> chunk_size);
+    ~TpassOnlineStream(){};
+
+    std::unique_ptr<VadModel> vad_online_handle = nullptr;
+    std::unique_ptr<Model> asr_online_handle = nullptr;
+};
+TpassOnlineStream* CreateTpassOnlineStream(void* tpass_stream, std::vector<int> chunk_size);
+} // namespace funasr
+#endif

runtime/onnxruntime/include/tpass-stream.h

@@ -0,0 +1,40 @@
+#ifndef TPASS_STREAM_H
+#define TPASS_STREAM_H
+
+#include <memory>
+#include <string>
+#include <map>
+#include "model.h"
+#include "punc-model.h"
+#include "vad-model.h"
+#if !defined(__APPLE__)
+#include "itn-model.h"
+#endif
+
+namespace funasr {
+class TpassStream {
+  public:
+    TpassStream(std::map<std::string, std::string>& model_path, int thread_num);
+    ~TpassStream(){};
+
+    std::unique_ptr<VadModel> vad_handle = nullptr;
+    std::unique_ptr<Model> asr_handle = nullptr;
+    std::unique_ptr<PuncModel> punc_online_handle = nullptr;
+#if !defined(__APPLE__)
+    std::unique_ptr<ITNModel> itn_handle = nullptr;
+#endif
+    bool UseVad(){return use_vad;};
+    bool UsePunc(){return use_punc;}; 
+    bool UseITN(){return use_itn;};
+    std::string GetModelType(){return model_type;};
+    
+  private:
+    bool use_vad=false;
+    bool use_punc=false;
+    bool use_itn=false;
+    std::string model_type = MODEL_PARA;
+};
+
+TpassStream *CreateTpassStream(std::map<std::string, std::string>& model_path, int thread_num=1);
+} // namespace funasr
+#endif

runtime/onnxruntime/include/vad-model.h

@@ -0,0 +1,21 @@
+
+#ifndef VAD_MODEL_H
+#define VAD_MODEL_H
+
+#include <string>
+#include <map>
+#include <vector>
+
+namespace funasr {
+class VadModel {
+  public:
+    virtual ~VadModel(){};
+    virtual void InitVad(const std::string &vad_model, const std::string &vad_cmvn, const std::string &vad_config, int thread_num)=0;
+    virtual std::vector<std::vector<int>> Infer(std::vector<float> &waves, bool input_finished=true)=0;
+    virtual int GetVadSampleRate() = 0;
+};
+
+VadModel *CreateVadModel(std::map<std::string, std::string>& model_path, int thread_num);
+VadModel *CreateVadModel(void* fsmnvad_handle);
+} // namespace funasr
+#endif

runtime/onnxruntime/readme.md

@@ -0,0 +1,60 @@
+# Please ref to [websocket service](https://github.com/modelscope/FunASR/tree/main/runtime/websocket)
+
+# If you want to compile the file yourself, you can follow the steps below.
+## Building for Linux/Unix
+### Download onnxruntime
+```shell
+wget https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/onnxruntime-linux-x64-1.14.0.tgz
+tar -zxvf onnxruntime-linux-x64-1.14.0.tgz
+```
+
+### Download ffmpeg
+```shell
+wget https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/ffmpeg-master-latest-linux64-gpl-shared.tar.xz
+tar -xvf ffmpeg-master-latest-linux64-gpl-shared.tar.xz
+```
+
+### Install deps
+```shell
+# openblas
+sudo apt-get install libopenblas-dev #ubuntu
+# sudo yum -y install openblas-devel #centos
+
+# openssl
+apt-get install libssl-dev #ubuntu 
+# yum install openssl-devel #centos
+```
+
+### Build runtime
+```shell
+git clone https://github.com/modelscope/FunASR.git && cd FunASR/runtime/onnxruntime
+mkdir build && cd build
+cmake  -DCMAKE_BUILD_TYPE=release .. -DONNXRUNTIME_DIR=/path/to/onnxruntime-linux-x64-1.14.0 -DFFMPEG_DIR=/path/to/ffmpeg-master-latest-linux64-gpl-shared
+make -j 4
+```
+
+
+## Building for Windows
+### Download onnxruntime
+https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/onnxruntime-win-x64-1.16.1.zip
+
+Download and unzip to d:/onnxruntime-win-x64-1.16.1
+
+### Download ffmpeg
+https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/ffmpeg-master-latest-win64-gpl-shared.zip
+
+Download and unzip to d:/ffmpeg-master-latest-win64-gpl-shared
+
+### Build runtime
+```
+git clone https://github.com/modelscope/FunASR.git
+cd FunASR/runtime/onnxruntime
+mkdir build
+cd build
+cmake ../ -D FFMPEG_DIR=d:/ffmpeg-master-latest-win64-gpl-shared -D ONNXRUNTIME_DIR=d:/onnxruntime-win-x64-1.16.1
+```
+
+Visual Studio open FunASR/runtime/onnxruntime/build/FunASROnnx.sln start build. 
+After compilation, the executable file is located here: FunASR/runtime/onnxruntime/build/bin/Debug.
+Copy the required DLL libraries from (onnxruntime-win-x64-1.16.1/lib, ffmpeg-master-latest-win64-gpl-shared/bin) to this location: FunASR/runtime/onnxruntime/build/bin/Debug
+

runtime/onnxruntime/src/CMakeLists.txt

@@ -0,0 +1,51 @@
+
+file(GLOB files1 "*.cpp")
+if(APPLE)
+    file(GLOB itn_files "itn-*.cpp")
+    list(REMOVE_ITEM files1 ${itn_files})
+endif(APPLE)
+list(REMOVE_ITEM files1 "${CMAKE_CURRENT_SOURCE_DIR}/paraformer-torch.cpp")
+set(files ${files1})
+
+if(GPU)
+    set(files ${files} "${CMAKE_CURRENT_SOURCE_DIR}/paraformer-torch.cpp")
+endif()
+
+message("files: ${files}")
+
+if(WIN32)
+add_compile_options("$<$<CXX_COMPILER_ID:MSVC>:/execution-charset:utf-8>")
+add_compile_options("$<$<CXX_COMPILER_ID:MSVC>:/source-charset:utf-8>")
+add_compile_options("$<$<CXX_COMPILER_ID:MSVC>:/bigobj>")
+endif()
+
+add_library(funasr SHARED ${files})
+
+if(WIN32)
+    set(EXTRA_LIBS yaml-cpp csrc kaldi-decoder fst glog gflags onnxruntime)
+    include_directories(${ONNXRUNTIME_DIR}/include)
+    target_link_directories(funasr PUBLIC ${ONNXRUNTIME_DIR}/lib)
+    target_compile_definitions(funasr PUBLIC -D_FUNASR_API_EXPORT -DNOMINMAX -DYAML_CPP_DLL)
+else()
+    set(EXTRA_LIBS pthread yaml-cpp csrc kaldi-decoder fst glog gflags)
+    include_directories(${ONNXRUNTIME_DIR}/include)
+    if(APPLE)
+        target_link_directories(funasr PUBLIC ${ONNXRUNTIME_DIR}/lib)
+    endif(APPLE)    
+endif()
+
+if(ENABLE_FFMPEG AND NOT APPLE)
+    target_compile_definitions(funasr PUBLIC -DENABLE_FFMPEG)
+    include_directories(${FFMPEG_DIR}/include)
+    target_link_directories(funasr PUBLIC ${FFMPEG_DIR}/lib)
+    target_link_libraries(funasr PUBLIC avutil avcodec avformat swresample)
+endif()
+
+if(GPU)
+    set(TORCH_DEPS torch torch_cuda torch_cpu c10 c10_cuda torch_blade ral_base_context)
+endif()
+
+#message("CXX_FLAGS "${CMAKE_CXX_FLAGS})
+include_directories(${CMAKE_SOURCE_DIR}/include)
+include_directories(${CMAKE_SOURCE_DIR}/third_party)
+target_link_libraries(funasr PUBLIC onnxruntime ${EXTRA_LIBS} ${TORCH_DEPS})

runtime/onnxruntime/src/alignedmem.cpp

@@ -0,0 +1,21 @@
+#include "precomp.h"
+
+namespace funasr {
+void *AlignedMalloc(size_t alignment, size_t required_bytes)
+{
+    void *p1;  // original block
+    void **p2; // aligned block
+    int offset = alignment - 1 + sizeof(void *);
+    if ((p1 = (void *)malloc(required_bytes + offset)) == nullptr) {
+        return nullptr;
+    }
+    p2 = (void **)(((size_t)(p1) + offset) & ~(alignment - 1));
+    p2[-1] = p1;
+    return p2;
+}
+
+void AlignedFree(void *p)
+{
+    free(((void **)p)[-1]);
+}
+} // namespace funasr

runtime/onnxruntime/src/alignedmem.h

@@ -0,0 +1,10 @@
+
+#ifndef ALIGNEDMEM_H
+#define ALIGNEDMEM_H
+
+namespace funasr {
+extern void *AlignedMalloc(size_t alignment, size_t required_bytes);
+extern void AlignedFree(void *p);
+
+} // namespace funasr
+#endif

runtime/onnxruntime/src/bias-lm.cpp

@@ -0,0 +1,180 @@
+#include "bias-lm.h"
+#ifdef _WIN32
+#include "fst-types.cc"
+#endif
+namespace funasr {
+void print(std::queue<StateId> &q) {
+  std::queue<StateId> data = q;
+  while (!data.empty())
+  {
+    cout << data.front() << " ";
+    data.pop();
+  }
+  cout << endl;
+}
+
+void BiasLm::LoadCfgFromYaml(const char* filename, BiasLmOption &opt) {
+  YAML::Node config;
+  try {
+    config = YAML::LoadFile(filename);
+  } catch(exception const &e) {
+    LOG(INFO) << "Error loading file, yaml file error or not exist.";
+    exit(-1);
+  }
+  try {
+    YAML::Node bias_lm_conf = config["bias_lm_conf"];
+    opt_.incre_bias_ = bias_lm_conf["increment_weight"].as<float>();
+  } catch(exception const &e) {
+  }
+}
+
+void BiasLm::BuildGraph(std::vector<std::vector<int>> &split_id_vec,
+  std::vector<float> &custom_weight) {
+  if (split_id_vec.empty()) {
+    LOG(INFO) << "Skip building biaslm graph, hotword not exits.";
+    return ; 
+  }
+  assert(split_id_vec.size() == custom_weight.size());
+  // Build prefix tree
+  std::unique_ptr<fst::StdVectorFst> prefix_tree(new fst::StdVectorFst());
+  StateId start_state = prefix_tree->AddState();
+  prefix_tree->SetStart(start_state);
+  int id = 0;
+  for (auto& x : split_id_vec) {
+    StateId state = start_state;
+    StateId next_state = state;
+    float w = custom_weight[id++];
+    std::vector<int> split_id = x;
+    for (int j = 0; j < split_id.size(); j++) {
+      next_state = prefix_tree->AddState();
+      if (j == split_id.size() - 1) {
+        prefix_tree->SetFinal(next_state, w);
+      }
+      prefix_tree->AddArc(state, Arc(split_id[j], split_id[j], opt_.incre_bias_, next_state));
+      state = next_state;
+    }
+  }
+  graph_ = std::unique_ptr<fst::StdVectorFst>(new fst::StdVectorFst());
+  fst::Determinize(*prefix_tree, graph_.get());
+
+  int num_node = graph_->NumStates();
+  node_list_.resize(num_node);
+  for (auto& x : split_id_vec) {
+    StateId cur_state = 0;
+    StateId next_state = 0;
+    std::vector<int> split_id = x;
+    for (int j = 0; j < split_id.size(); j++) {
+      Matcher matcher(*graph_, fst::MATCH_INPUT);
+      matcher.SetState(cur_state);
+      if (matcher.Find(split_id[j])) {
+        next_state = matcher.Value().nextstate;
+        if (graph_->Final(next_state) != Weight::Zero()) {
+          node_list_[next_state].is_final_ = true;
+        }
+        node_list_[next_state].score_ = opt_.incre_bias_ * (j + 1);
+        cur_state = next_state;
+      }
+    }
+  }
+  
+  // Build Aho-Corasick Automata
+  std::queue<StateId> q;
+  Matcher matcher(*graph_, fst::MATCH_INPUT);
+  // Back off state of all child nodes of the root node points to the root node
+  for (ArcIterator aiter(*graph_, start_state); !aiter.Done(); aiter.Next()) {
+    const Arc& arc = aiter.Value();
+    node_list_[arc.nextstate].back_off_ = start_state;
+    float back_off_score = (node_list_[arc.nextstate].is_final_ ? 0 :
+      node_list_[start_state].score_ - node_list_[arc.nextstate].score_);
+    graph_->AddArc(arc.nextstate, Arc(0, 0, back_off_score, start_state));
+    q.push(arc.nextstate);
+  }
+  while (!q.empty()) {
+    StateId state_id = q.front();
+    q.pop();
+    for (ArcIterator aiter(*graph_, state_id); !aiter.Done(); aiter.Next()) {
+      const Arc& arc = aiter.Value();
+      StateId next_state = arc.nextstate;
+      StateId temp_state = node_list_[state_id].back_off_;
+      if (next_state == start_state || next_state == temp_state) { 
+        continue; 
+      }
+      while (true) {
+        matcher.SetState(temp_state);
+        if (matcher.Find(arc.ilabel)) {
+          node_list_[next_state].back_off_ = matcher.Value().nextstate;
+          break;
+        } else if (temp_state == start_state) {
+          node_list_[next_state].back_off_ = start_state;
+          break;
+        }
+        temp_state = node_list_[temp_state].back_off_;
+      }
+      float back_off_score = (node_list_[next_state].is_final_ ? 0 :
+        node_list_[node_list_[next_state].back_off_].score_ -
+        node_list_[next_state].score_);
+      graph_->AddArc(next_state, Arc(0, 0, back_off_score, 
+        node_list_[next_state].back_off_));
+      q.push(next_state);
+    }
+  }
+  fst::ArcSort(graph_.get(), fst::StdILabelCompare());
+  //graph_->Write("graph.final.fst");
+}
+
+float BiasLm::BiasLmScore(const StateId &his_state, const Label &lab, Label &new_state) {
+  if (lab < 1 || lab > phn_set_.Size() || !graph_) { return VALUE_ZERO; }
+  StateId cur_state = his_state;
+  StateId next_state;
+  float score = VALUE_ZERO;
+  Matcher matcher(*graph_, fst::MATCH_INPUT);
+  while (true) {
+    StateId prev_state = cur_state;
+    matcher.SetState(cur_state);
+    if (matcher.Find(lab)) {
+      next_state = matcher.Value().nextstate;
+      score += matcher.Value().weight.Value();
+      if (node_list_[next_state].is_final_) {
+        score = score + graph_->Final(next_state).Value();
+      }
+      cur_state = next_state;
+      break;
+    } else {
+      ArcIterator aiter(*graph_, cur_state);
+      const Arc& arc = aiter.Value();
+      if (arc.ilabel == 0) {
+        score += arc.weight.Value();
+        next_state = arc.nextstate;
+        cur_state = next_state;
+      }
+      if (prev_state == ROOT_NODE && cur_state == ROOT_NODE) {
+        break;
+      }
+    }
+  }
+  new_state = cur_state;
+  return score;
+}
+
+void BiasLm::VocabIdToPhnIdVector(int vocab_id, std::vector<int> &phn_ids) {
+  bool is_oov = false;
+  phn_ids.clear();
+  std::string word = vocab_.Id2String(vocab_id);
+  std::vector<std::string> phn_vec;
+  Utf8ToCharset(word, phn_vec);
+  for (auto& phn : phn_vec) {
+    if (!phn_set_.Find(phn)) {
+      is_oov = true;
+      break;
+    } else {
+      phn_ids.push_back(phn_set_.String2Id(phn));
+    }
+  }
+  if (is_oov) { phn_ids.clear(); }
+}
+
+std::string BiasLm::GetPhoneLabel(int phone_id) {
+  if (phone_id < 0 || phone_id >= phn_set_.Size()) { return ""; }
+  return phn_set_.Id2String(phone_id);
+}
+}

runtime/onnxruntime/src/bias-lm.h

@@ -0,0 +1,150 @@
+#ifndef BIAS_LM_
+#define BIAS_LM_
+#include <assert.h>
+#include "util.h"
+#include "fst/fstlib.h"
+#include "phone-set.h"
+#include "vocab.h"
+#include "util/text-utils.h"
+#include <yaml-cpp/yaml.h>
+#ifdef _WIN32
+#include "win_func.h"
+#endif
+// node type
+#define ROOT_NODE 0
+#define VALUE_ZERO 0.0f
+
+namespace funasr {
+typedef fst::StdArc Arc;
+typedef typename Arc::StateId StateId;
+typedef typename Arc::Weight Weight;
+typedef typename Arc::Label Label;
+typedef typename fst::SortedMatcher<fst::StdVectorFst> Matcher;
+typedef typename fst::ArcIterator<fst::StdVectorFst> ArcIterator;
+
+class Node {
+ public:
+  Node() : score_(0.0f), is_final_(false), back_off_(-1) {}
+  float score_;
+  bool is_final_;
+  StateId back_off_;
+};
+
+class BiasLmOption {
+ public:
+  BiasLmOption() : incre_bias_(20.0f), scale_(1.0f) {}
+  float incre_bias_;
+  float scale_;
+};
+
+class BiasLm {
+ public:
+  BiasLm(const string &hws_file, const string &cfg_file, 
+    const PhoneSet& phn_set, const Vocab& vocab) :
+    phn_set_(phn_set), vocab_(vocab) {
+    std::string line;
+    std::ifstream ifs_hws(hws_file.c_str());
+    std::vector<float> custom_weight;
+    std::vector<std::vector<int>> split_id_vec;
+
+    struct timeval start, end;
+    gettimeofday(&start, nullptr);
+
+    LoadCfgFromYaml(cfg_file.c_str(), opt_);
+    while (getline(ifs_hws, line)) {
+      Trim(&line);
+      if (line.empty()) {
+        continue;
+      }
+      float score = 1.0f;
+      bool is_oov = false;
+      std::vector<std::string> text;
+      std::vector<std::string> split_str;
+      std::vector<int> split_id;
+      SplitStringToVector(line, "\t", true, &text);
+      if (text.size() > 1) {
+        score = std::stof(text[1]);
+      }
+      SplitChiEngCharacters(text[0], split_str);
+      for (auto &str : split_str) {
+        std::vector<string> lex_vec;
+        std::string lex_str = vocab_.Word2Lex(str);
+        SplitStringToVector(lex_str, " ", true, &lex_vec);
+        for (auto &token : lex_vec) {
+          split_id.push_back(phn_set_.String2Id(token));
+          if (!phn_set_.Find(token)) {
+            is_oov = true;
+            break;
+          }
+        }
+      }
+      if (!is_oov) {
+        split_id_vec.push_back(split_id);
+        custom_weight.push_back(score);
+      }
+    }
+    BuildGraph(split_id_vec, custom_weight);
+    ifs_hws.close();
+
+    gettimeofday(&end, nullptr);
+    long seconds = (end.tv_sec - start.tv_sec);
+    long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+    LOG(INFO) << "Build bias lm takes " << (double)modle_init_micros / 1000000 << " s";
+  }
+
+  BiasLm(unordered_map<string, int> &hws_map, int inc_bias,
+    const PhoneSet& phn_set, const Vocab& vocab) :
+    phn_set_(phn_set), vocab_(vocab) {
+    std::vector<float> custom_weight;
+    std::vector<std::vector<int>> split_id_vec;
+
+    struct timeval start, end;
+    gettimeofday(&start, nullptr);
+    opt_.incre_bias_ = inc_bias;
+    for (const pair<string, int>& kv : hws_map) {
+      float score = 1.0f;
+      bool is_oov = false;
+      std::vector<std::string> text;
+      std::vector<std::string> split_str;
+      std::vector<int> split_id;
+      score = kv.second;
+      SplitChiEngCharacters(kv.first, split_str);
+      for (auto &str : split_str) {
+        std::vector<string> lex_vec;
+        std::string lex_str = vocab_.Word2Lex(str);
+        SplitStringToVector(lex_str, " ", true, &lex_vec);
+        for (auto &token : lex_vec) {
+          split_id.push_back(phn_set_.String2Id(token));
+          if (!phn_set_.Find(token)) {
+            is_oov = true;
+            break;
+          }
+        }
+      }
+      if (!is_oov) {
+        split_id_vec.push_back(split_id);
+        custom_weight.push_back(score);
+      }
+    }
+    BuildGraph(split_id_vec, custom_weight);
+
+    gettimeofday(&end, nullptr);
+    long seconds = (end.tv_sec - start.tv_sec);
+    long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
+    LOG(INFO) << "Build bias lm takes " << (double)modle_init_micros / 1000000 << " s";
+  }
+
+  void BuildGraph(std::vector<std::vector<int>> &vec, std::vector<float> &wts);
+  float BiasLmScore(const StateId &cur_state, const Label &lab, Label &new_state);
+  void VocabIdToPhnIdVector(int vocab_id, std::vector<int> &phn_ids);
+  void LoadCfgFromYaml(const char* filename, BiasLmOption &opt);
+  std::string GetPhoneLabel(int phone_id);
+ private:
+  const PhoneSet& phn_set_;
+  const Vocab& vocab_;
+  std::unique_ptr<fst::StdVectorFst> graph_ = nullptr;
+  std::vector<Node> node_list_;
+  BiasLmOption opt_;
+};
+} // namespace funasr
+#endif // BIAS_LM_

runtime/onnxruntime/src/common-struct.h

@@ -0,0 +1,6 @@
+
+#ifndef COMMONSTRUCT_H
+#define COMMONSTRUCT_H
+
+
+#endif

runtime/onnxruntime/src/commonfunc.h

@@ -0,0 +1,111 @@
+#pragma once 
+#include <algorithm>
+#ifdef _WIN32
+#include <codecvt>
+#endif
+
+namespace funasr {
+typedef struct
+{
+    std::string msg;
+    std::string stamp;
+    std::string stamp_sents;
+    std::string tpass_msg;
+    float snippet_time;
+    int64_t start = 0;
+    int64_t end = 0;
+}FUNASR_RECOG_RESULT;
+
+typedef struct
+{
+    std::vector<std::vector<int>>* segments;
+    float  snippet_time;
+}FUNASR_VAD_RESULT;
+
+typedef struct
+{
+    string msg;
+    vector<string> arr_cache;
+}FUNASR_PUNC_RESULT;
+
+#ifdef _WIN32
+
+#define ORTSTRING(str) StrToWstr(str)
+#define ORTCHAR(str) StrToWstr(str).c_str()
+
+inline std::wstring String2wstring(const std::string& str, const std::string& locale)
+{
+    typedef std::codecvt_byname<wchar_t, char, std::mbstate_t> F;
+    std::wstring_convert<F> strCnv(new F(locale));
+    return strCnv.from_bytes(str);
+}
+
+inline std::wstring  StrToWstr(std::string str) {
+    if (str.length() == 0)
+        return L"";
+    return  String2wstring(str, "zh-CN");
+
+}
+
+#else
+
+#define ORTSTRING(str) str
+#define ORTCHAR(str) str
+
+#endif
+
+
+
+inline void GetInputName(Ort::Session* session, string& inputName,int nIndex=0) {
+    size_t numInputNodes = session->GetInputCount();
+    if (numInputNodes > 0) {
+        Ort::AllocatorWithDefaultOptions allocator;
+        {
+            auto t = session->GetInputNameAllocated(nIndex, allocator);
+            inputName = t.get();
+        }
+    }
+}
+
+inline void GetInputNames(Ort::Session* session, std::vector<std::string> &m_strInputNames,
+                   std::vector<const char *> &m_szInputNames) {
+    Ort::AllocatorWithDefaultOptions allocator;
+    size_t numNodes = session->GetInputCount();
+    m_strInputNames.resize(numNodes);
+    m_szInputNames.resize(numNodes);
+    for (size_t i = 0; i != numNodes; ++i) {    
+        auto t = session->GetInputNameAllocated(i, allocator);
+        m_strInputNames[i] = t.get();
+        m_szInputNames[i] = m_strInputNames[i].c_str();
+    }
+}
+
+inline void GetOutputName(Ort::Session* session, string& outputName, int nIndex = 0) {
+    size_t numOutputNodes = session->GetOutputCount();
+    if (numOutputNodes > 0) {
+        Ort::AllocatorWithDefaultOptions allocator;
+        {
+            auto t = session->GetOutputNameAllocated(nIndex, allocator);
+            outputName = t.get();
+        }
+    }
+}
+
+inline void GetOutputNames(Ort::Session* session, std::vector<std::string> &m_strOutputNames,
+                   std::vector<const char *> &m_szOutputNames) {
+    Ort::AllocatorWithDefaultOptions allocator;
+    size_t numNodes = session->GetOutputCount();
+    m_strOutputNames.resize(numNodes);
+    m_szOutputNames.resize(numNodes);
+    for (size_t i = 0; i != numNodes; ++i) {    
+        auto t = session->GetOutputNameAllocated(i, allocator);
+        m_strOutputNames[i] = t.get();
+        m_szOutputNames[i] = m_strOutputNames[i].c_str();
+    }
+}
+
+template <class ForwardIterator>
+inline static size_t Argmax(ForwardIterator first, ForwardIterator last) {
+    return std::distance(first, std::max_element(first, last));
+}
+} // namespace funasr

runtime/onnxruntime/src/ct-transformer-online.cpp

@@ -0,0 +1,274 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#include "precomp.h"
+
+namespace funasr {
+CTTransformerOnline::CTTransformerOnline()
+:env_(ORT_LOGGING_LEVEL_ERROR, ""),session_options{}
+{
+}
+
+void CTTransformerOnline::InitPunc(const std::string &punc_model, const std::string &punc_config, const std::string &token_file, int thread_num){
+    session_options.SetIntraOpNumThreads(thread_num);
+    session_options.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
+    session_options.DisableCpuMemArena();
+
+    try{
+        m_session = std::make_unique<Ort::Session>(env_, ORTSTRING(punc_model).c_str(), session_options);
+        LOG(INFO) << "Successfully load model from " << punc_model;
+    }
+    catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load punc onnx model: " << e.what();
+        exit(-1);
+    }
+    // read inputnames outputnames
+    GetInputNames(m_session.get(), m_strInputNames, m_szInputNames);
+    GetOutputNames(m_session.get(), m_strOutputNames, m_szOutputNames);
+
+	m_tokenizer.OpenYaml(punc_config.c_str(), token_file.c_str());
+	m_tokenizer.JiebaInit(punc_config);
+}
+
+CTTransformerOnline::~CTTransformerOnline()
+{
+}
+
+string CTTransformerOnline::AddPunc(const char* sz_input, vector<string> &arr_cache, std::string language)
+{
+    string strResult;
+    vector<string> strOut;
+    vector<int> InputData;
+    string strText; //full_text
+    strText = accumulate(arr_cache.begin(), arr_cache.end(), strText);
+
+    // 如果上一句的结尾是英语字母，并且这一句的开始也是英语字母，应该添加空格
+    if ((strText.size() > 0 && !(strText[strText.size()-1] & 0x80)) && (strlen(sz_input) > 0 && !(sz_input[0] & 0x80)))
+        strText += " ";
+
+    strText += sz_input;  // full_text = precache + text  
+    m_tokenizer.Tokenize(strText.c_str(), strOut, InputData);
+
+    int nTotalBatch = ceil((float)InputData.size() / TOKEN_LEN);
+    int nCurBatch = -1;
+    int nSentEnd = -1, nLastCommaIndex = -1;
+    vector<int32_t> RemainIDs; // 
+    vector<string> RemainStr; //
+    vector<int>     new_mini_sentence_punc; //          sentence_punc_list = []
+    vector<string> sentenceOut; // sentenceOut
+    vector<string> sentence_punc_list,sentence_words_list,sentence_punc_list_out; // sentence_words_list = []
+    
+    int nSkipNum = 0;
+    int nDiff = 0;
+    for (size_t i = 0; i < InputData.size(); i += TOKEN_LEN)
+    {
+        nDiff = (i + TOKEN_LEN) < InputData.size() ? (0) : (i + TOKEN_LEN - InputData.size());
+        vector<int32_t> InputIDs(InputData.begin() + i, InputData.begin() + i + (TOKEN_LEN - nDiff));
+        vector<string> InputStr(strOut.begin() + i, strOut.begin() + i + (TOKEN_LEN - nDiff));
+        InputIDs.insert(InputIDs.begin(), RemainIDs.begin(), RemainIDs.end()); // RemainIDs+InputIDs;
+        InputStr.insert(InputStr.begin(), RemainStr.begin(), RemainStr.end()); // RemainStr+InputStr;
+
+        auto Punction = Infer(InputIDs, arr_cache.size());
+        nCurBatch = i / TOKEN_LEN;
+        if (nCurBatch < nTotalBatch - 1) // not the last minisetence
+        {
+            nSentEnd = -1;
+            nLastCommaIndex = -1;
+            for (int nIndex = Punction.size() - 2; nIndex > 0; nIndex--)
+            {
+                if (m_tokenizer.Id2Punc(Punction[nIndex]) == m_tokenizer.Id2Punc(PERIOD_INDEX) || m_tokenizer.Id2Punc(Punction[nIndex]) == m_tokenizer.Id2Punc(QUESTION_INDEX))
+                {
+                    nSentEnd = nIndex;
+                    break;
+                }
+                if (nLastCommaIndex < 0 && m_tokenizer.Id2Punc(Punction[nIndex]) == m_tokenizer.Id2Punc(COMMA_INDEX))
+                {
+                    nLastCommaIndex = nIndex;
+                }
+            }
+            if (nSentEnd < 0 && InputStr.size() > CACHE_POP_TRIGGER_LIMIT && nLastCommaIndex > 0)
+            {
+                nSentEnd = nLastCommaIndex;
+                Punction[nSentEnd] = PERIOD_INDEX;
+            }
+            RemainStr.assign(InputStr.begin() + (nSentEnd + 1), InputStr.end());
+            RemainIDs.assign(InputIDs.begin() + (nSentEnd + 1), InputIDs.end());
+            InputStr.assign(InputStr.begin(), InputStr.begin() + (nSentEnd + 1));  // minit_sentence
+            Punction.assign(Punction.begin(), Punction.begin() + (nSentEnd + 1));
+        }
+        
+        for (auto& item : Punction)  
+        {
+            sentence_punc_list.push_back(m_tokenizer.Id2Punc(item));
+        }
+
+        sentence_words_list.insert(sentence_words_list.end(), InputStr.begin(), InputStr.end());
+
+        new_mini_sentence_punc.insert(new_mini_sentence_punc.end(), Punction.begin(), Punction.end());
+    }    
+    vector<string> WordWithPunc;
+    for (int i = 0; i < sentence_words_list.size(); i++) // for i in range(0, len(sentence_words_list)):
+    {
+        if (!(sentence_words_list[i][0] & 0x80) && (i + 1) < sentence_words_list.size() && !(sentence_words_list[i + 1][0] & 0x80))
+        {
+            sentence_words_list[i] = sentence_words_list[i] + " ";
+        }
+        if (nSkipNum < arr_cache.size())  //    if skip_num < len(cache):
+            nSkipNum++;
+        else
+            WordWithPunc.push_back(sentence_words_list[i]);
+
+        if (nSkipNum >= arr_cache.size())
+        {
+            sentence_punc_list_out.push_back(sentence_punc_list[i]);
+            if (sentence_punc_list[i] != NOTPUNC)
+            {
+                WordWithPunc.push_back(sentence_punc_list[i]);
+            }
+        }
+    }
+
+    sentenceOut.insert(sentenceOut.end(), WordWithPunc.begin(), WordWithPunc.end()); //
+    nSentEnd = -1;
+    for (int i = sentence_punc_list.size() - 2; i > 0; i--)
+    {
+        if (new_mini_sentence_punc[i] == PERIOD_INDEX || new_mini_sentence_punc[i] == QUESTION_INDEX)
+        {
+            nSentEnd = i;
+            break;
+        }
+    }
+    arr_cache.assign(sentence_words_list.begin() + (nSentEnd + 1), sentence_words_list.end());
+
+    if (sentenceOut.size() > 0 && m_tokenizer.IsPunc(sentenceOut[sentenceOut.size() - 1]))
+    {
+        sentenceOut.assign(sentenceOut.begin(), sentenceOut.end() - 1);
+        sentence_punc_list_out[sentence_punc_list_out.size() - 1] = m_tokenizer.Id2Punc(NOTPUNC_INDEX);
+    }
+    return accumulate(sentenceOut.begin(), sentenceOut.end(), string(""));
+}
+
+vector<int> CTTransformerOnline::Infer(vector<int32_t> input_data, int nCacheSize)
+{
+    Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
+    vector<int> punction;
+    std::array<int64_t, 2> input_shape_{ 1, (int64_t)input_data.size()};
+    Ort::Value onnx_input = Ort::Value::CreateTensor(
+        m_memoryInfo,
+        input_data.data(),
+        input_data.size() * sizeof(int32_t),
+        input_shape_.data(),
+        input_shape_.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32);
+
+    std::array<int32_t,1> text_lengths{ (int32_t)input_data.size() };
+    std::array<int64_t,1> text_lengths_dim{ 1 };
+    Ort::Value onnx_text_lengths = Ort::Value::CreateTensor<int32_t>(
+        m_memoryInfo,
+        text_lengths.data(),
+        text_lengths.size(),
+        text_lengths_dim.data(),
+        text_lengths_dim.size()); //, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32);
+
+    //vad_mask
+    // vector<float> arVadMask,arSubMask;
+    vector<float> arVadMask;
+    int nTextLength = input_data.size();
+
+    VadMask(nTextLength, nCacheSize, arVadMask);
+    // Triangle(nTextLength, arSubMask);
+    std::array<int64_t, 4> VadMask_Dim{ 1,1, nTextLength ,nTextLength };
+    Ort::Value onnx_vad_mask = Ort::Value::CreateTensor<float>(
+        m_memoryInfo,
+        arVadMask.data(),
+        arVadMask.size(), // * sizeof(float),
+        VadMask_Dim.data(),
+        VadMask_Dim.size()); // , ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT);
+    //sub_masks
+    
+    std::array<int64_t, 4> SubMask_Dim{ 1,1, nTextLength ,nTextLength };
+    Ort::Value onnx_sub_mask = Ort::Value::CreateTensor<float>(
+        m_memoryInfo,
+        arVadMask.data(),
+        arVadMask.size(),
+        SubMask_Dim.data(),
+        SubMask_Dim.size()); // , ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT);
+
+    std::vector<Ort::Value> input_onnx;
+    input_onnx.emplace_back(std::move(onnx_input));
+    input_onnx.emplace_back(std::move(onnx_text_lengths));
+    input_onnx.emplace_back(std::move(onnx_vad_mask));
+    input_onnx.emplace_back(std::move(onnx_sub_mask));
+        
+    try {
+        auto outputTensor = m_session->Run(Ort::RunOptions{nullptr}, m_szInputNames.data(), input_onnx.data(), m_szInputNames.size(), m_szOutputNames.data(), m_szOutputNames.size());
+        std::vector<int64_t> outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape();
+
+        int64_t outputCount = std::accumulate(outputShape.begin(), outputShape.end(), 1, std::multiplies<int64_t>());
+        float * floatData = outputTensor[0].GetTensorMutableData<float>();
+
+        for (int i = 0; i < outputCount; i += CANDIDATE_NUM)
+        {
+            int index = Argmax(floatData + i, floatData + i + CANDIDATE_NUM-1);
+            punction.push_back(index);
+        }
+    }
+    catch (std::exception const &e)
+    {
+        LOG(ERROR) << "Error when run punc onnx forword: " << (e.what());
+    }
+    return punction;
+}
+
+void CTTransformerOnline::VadMask(int nSize, int vad_pos, vector<float>& Result)
+{
+    Result.resize(0);
+    Result.assign(nSize * nSize, 1);
+    if (vad_pos <= 0 || vad_pos >= nSize)
+    {
+        return;
+    }
+    for (int i = 0; i < vad_pos-1; i++)
+    {
+        for (int j = vad_pos; j < nSize; j++)
+        {
+            Result[i * nSize + j] = 0.0f;
+        }
+    }
+}
+
+void CTTransformerOnline::Triangle(int text_length, vector<float>& Result)
+{
+    Result.resize(0);
+    Result.assign(text_length * text_length,1); // generate a zeros: text_length x text_length
+
+    for (int i = 0; i < text_length; i++) // rows
+    {
+        for (int j = i+1; j<text_length; j++) //cols
+        {
+            Result[i * text_length + j] = 0.0f;
+        }
+
+    }
+    //Transport(Result, text_length, text_length);
+}
+
+void CTTransformerOnline::Transport(vector<float>& In,int nRows, int nCols)
+{
+    vector<float> Out;
+    Out.resize(nRows * nCols);
+    int i = 0;
+    for (int j = 0; j < nCols; j++) {
+        for (; i < nRows * nCols; i++) {
+            Out[i] = In[j + nCols * (i % nRows)];
+            if ((i + 1) % nRows == 0) {
+                i++;
+                break;
+            }
+        }
+    }
+    In = Out;
+}
+
+} // namespace funasr

runtime/onnxruntime/src/ct-transformer-online.h

@@ -0,0 +1,37 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#pragma once 
+
+namespace funasr {
+class CTTransformerOnline : public PuncModel {
+/**
+ * Author: Speech Lab of DAMO Academy, Alibaba Group
+ * CT-Transformer: Controllable time-delay transformer for real-time punctuation prediction and disfluency detection
+ * https://arxiv.org/pdf/2003.01309.pdf
+*/
+
+private:
+
+	CTokenizer m_tokenizer;
+	vector<string> m_strInputNames, m_strOutputNames;
+	vector<const char*> m_szInputNames;
+	vector<const char*> m_szOutputNames;
+
+	std::shared_ptr<Ort::Session> m_session;
+    Ort::Env env_;
+    Ort::SessionOptions session_options;
+public:
+
+	CTTransformerOnline();
+	void InitPunc(const std::string &punc_model, const std::string &punc_config, const std::string &token_file, int thread_num);
+	~CTTransformerOnline();
+	vector<int>  Infer(vector<int32_t> input_data, int nCacheSize);
+	string AddPunc(const char* sz_input, vector<string> &arr_cache, std::string language="zh-cn");
+	void Transport(vector<float>& In, int nRows, int nCols);
+	void VadMask(int size, int vad_pos,vector<float>& Result);
+	void Triangle(int text_length, vector<float>& Result);
+};
+} // namespace funasr

runtime/onnxruntime/src/ct-transformer.cpp

@@ -0,0 +1,201 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#include "precomp.h"
+
+namespace funasr {
+CTTransformer::CTTransformer()
+:env_(ORT_LOGGING_LEVEL_ERROR, ""),session_options{}
+{
+}
+
+void CTTransformer::InitPunc(const std::string &punc_model, const std::string &punc_config, const std::string &token_file, int thread_num){
+    session_options.SetIntraOpNumThreads(thread_num);
+    session_options.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
+    session_options.DisableCpuMemArena();
+
+    try{
+        m_session = std::make_unique<Ort::Session>(env_, ORTSTRING(punc_model).c_str(), session_options);
+        LOG(INFO) << "Successfully load model from " << punc_model;
+    }
+    catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load punc onnx model: " << e.what();
+        exit(-1);
+    }
+    // read inputnames outputnames
+    GetInputNames(m_session.get(), m_strInputNames, m_szInputNames);
+    GetOutputNames(m_session.get(), m_strOutputNames, m_szOutputNames);
+    
+	m_tokenizer.OpenYaml(punc_config.c_str(), token_file.c_str());
+    m_tokenizer.JiebaInit(punc_config);
+}
+
+CTTransformer::~CTTransformer()
+{
+}
+
+string CTTransformer::AddPunc(const char* sz_input, std::string language)
+{
+    string strResult;
+    vector<string> strOut;
+    vector<int> InputData;
+    m_tokenizer.Tokenize(sz_input, strOut, InputData); 
+
+    int nTotalBatch = ceil((float)InputData.size() / TOKEN_LEN);
+    int nCurBatch = -1;
+    int nSentEnd = -1, nLastCommaIndex = -1;
+    vector<int32_t> RemainIDs; // 
+    vector<string> RemainStr; //
+    vector<int> NewPunctuation; //
+    vector<string> NewString; //
+    vector<string> NewSentenceOut;
+    vector<int> NewPuncOut;
+    int nDiff = 0;
+    for (size_t i = 0; i < InputData.size(); i += TOKEN_LEN)
+    {
+        nDiff = (i + TOKEN_LEN) < InputData.size() ? (0) : (i + TOKEN_LEN - InputData.size());
+        vector<int32_t> InputIDs(InputData.begin() + i, InputData.begin() + i + (TOKEN_LEN - nDiff));
+        vector<string> InputStr(strOut.begin() + i, strOut.begin() + i + (TOKEN_LEN - nDiff));
+        InputIDs.insert(InputIDs.begin(), RemainIDs.begin(), RemainIDs.end()); // RemainIDs+InputIDs;
+        InputStr.insert(InputStr.begin(), RemainStr.begin(), RemainStr.end()); // RemainStr+InputStr;
+
+        auto Punction = Infer(InputIDs);
+        nCurBatch = i / TOKEN_LEN;
+        if (nCurBatch < nTotalBatch - 1) // not the last minisetence
+        {
+            nSentEnd = -1;
+            nLastCommaIndex = -1;
+            for (int nIndex = Punction.size() - 2; nIndex > 0; nIndex--)
+            {
+                if (m_tokenizer.Id2Punc(Punction[nIndex]) == m_tokenizer.Id2Punc(PERIOD_INDEX) || m_tokenizer.Id2Punc(Punction[nIndex]) == m_tokenizer.Id2Punc(QUESTION_INDEX))
+                {
+                    nSentEnd = nIndex;
+                    break;
+                }
+                if (nLastCommaIndex < 0 && m_tokenizer.Id2Punc(Punction[nIndex]) == m_tokenizer.Id2Punc(COMMA_INDEX))
+                {
+                    nLastCommaIndex = nIndex;
+                }
+            }
+            if (nSentEnd < 0 && InputStr.size() > CACHE_POP_TRIGGER_LIMIT && nLastCommaIndex > 0)
+            {
+                nSentEnd = nLastCommaIndex;
+                Punction[nSentEnd] = PERIOD_INDEX;
+            }
+            RemainStr.assign(InputStr.begin() + (nSentEnd + 1), InputStr.end());
+            RemainIDs.assign(InputIDs.begin() + (nSentEnd + 1), InputIDs.end());
+            InputStr.assign(InputStr.begin(), InputStr.begin() + (nSentEnd + 1));  // minit_sentence
+            Punction.assign(Punction.begin(), Punction.begin() + (nSentEnd + 1));
+        }
+        
+        NewPunctuation.insert(NewPunctuation.end(), Punction.begin(), Punction.end());
+        vector<string> WordWithPunc;
+        for (int i = 0; i < InputStr.size(); i++)
+        {
+            // if (i > 0 && !(InputStr[i][0] & 0x80) && (i + 1) <InputStr.size() && !(InputStr[i+1][0] & 0x80))// �м��Ӣ�ģ�
+            if (i > 0 && !(InputStr[i-1][0] & 0x80) && !(InputStr[i][0] & 0x80))
+            {
+                InputStr[i] = " " + InputStr[i];
+            }
+            WordWithPunc.push_back(InputStr[i]);
+
+            if (Punction[i] != NOTPUNC_INDEX) // �»���
+            {
+                WordWithPunc.push_back(m_tokenizer.Id2Punc(Punction[i]));
+            }
+        }
+
+        NewString.insert(NewString.end(), WordWithPunc.begin(), WordWithPunc.end()); // new_mini_sentence += "".join(words_with_punc)
+        NewSentenceOut = NewString;
+        NewPuncOut = NewPunctuation;
+        // last mini sentence
+        if(nCurBatch == nTotalBatch - 1)
+        {
+            if (NewString[NewString.size() - 1] == m_tokenizer.Id2Punc(COMMA_INDEX) || NewString[NewString.size() - 1] == m_tokenizer.Id2Punc(DUN_INDEX))
+            {
+                NewSentenceOut.assign(NewString.begin(), NewString.end() - 1);
+                NewSentenceOut.push_back(m_tokenizer.Id2Punc(PERIOD_INDEX));
+                NewPuncOut.assign(NewPunctuation.begin(), NewPunctuation.end() - 1);
+                NewPuncOut.push_back(PERIOD_INDEX);
+            }
+            else if (NewString[NewString.size() - 1] != m_tokenizer.Id2Punc(PERIOD_INDEX) && NewString[NewString.size() - 1] != m_tokenizer.Id2Punc(QUESTION_INDEX))
+            {
+                NewSentenceOut = NewString;
+                NewSentenceOut.push_back(m_tokenizer.Id2Punc(PERIOD_INDEX));
+                NewPuncOut = NewPunctuation;
+                NewPuncOut.push_back(PERIOD_INDEX);
+            }
+        }
+    }
+
+    for (auto& item : NewSentenceOut){
+        strResult += item;
+    }
+    
+    if(language == "en-bpe"){
+        std::vector<std::string> chineseSymbols;
+        chineseSymbols.push_back("，");
+        chineseSymbols.push_back("。");
+        chineseSymbols.push_back("、");
+        chineseSymbols.push_back("？");
+
+        std::string englishSymbols = ",.,?";
+        for (size_t i = 0; i < chineseSymbols.size(); i++) {
+            size_t pos = 0;
+            while ((pos = strResult.find(chineseSymbols[i], pos)) != std::string::npos) {
+                strResult.replace(pos, 3, 1, englishSymbols[i]);
+                pos++;
+            }
+        }
+    }
+
+    return strResult;
+}
+
+vector<int> CTTransformer::Infer(vector<int32_t> input_data)
+{
+    Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
+    vector<int> punction;
+    std::array<int64_t, 2> input_shape_{ 1, (int64_t)input_data.size()};
+    Ort::Value onnx_input = Ort::Value::CreateTensor<int32_t>(
+        m_memoryInfo,
+        input_data.data(),
+        input_data.size(),
+        input_shape_.data(),
+        input_shape_.size());
+
+    std::array<int32_t,1> text_lengths{ (int32_t)input_data.size() };
+    std::array<int64_t,1> text_lengths_dim{ 1 };
+    Ort::Value onnx_text_lengths = Ort::Value::CreateTensor(
+        m_memoryInfo,
+        text_lengths.data(),
+        text_lengths.size() * sizeof(int32_t),
+        text_lengths_dim.data(),
+        text_lengths_dim.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32);
+    std::vector<Ort::Value> input_onnx;
+    input_onnx.emplace_back(std::move(onnx_input));
+    input_onnx.emplace_back(std::move(onnx_text_lengths));
+        
+    try {
+        auto outputTensor = m_session->Run(Ort::RunOptions{nullptr}, m_szInputNames.data(), input_onnx.data(), m_szInputNames.size(), m_szOutputNames.data(), m_szOutputNames.size());
+        std::vector<int64_t> outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape();
+
+        int64_t outputCount = std::accumulate(outputShape.begin(), outputShape.end(), 1, std::multiplies<int64_t>());
+        float * floatData = outputTensor[0].GetTensorMutableData<float>();
+
+        for (int i = 0; i < outputCount; i += CANDIDATE_NUM)
+        {
+            int index = Argmax(floatData + i, floatData + i + CANDIDATE_NUM-1);
+            punction.push_back(index);
+        }
+    }
+    catch (std::exception const &e)
+    {
+        LOG(ERROR) << "Error when run punc onnx forword: " << (e.what());
+    }
+    return punction;
+}
+
+} // namespace funasr

runtime/onnxruntime/src/ct-transformer.h

@@ -0,0 +1,34 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#pragma once 
+
+namespace funasr {
+class CTTransformer : public PuncModel {
+/**
+ * Author: Speech Lab of DAMO Academy, Alibaba Group
+ * CT-Transformer: Controllable time-delay transformer for real-time punctuation prediction and disfluency detection
+ * https://arxiv.org/pdf/2003.01309.pdf
+*/
+
+private:
+
+	CTokenizer m_tokenizer;
+	vector<string> m_strInputNames, m_strOutputNames;
+	vector<const char*> m_szInputNames;
+	vector<const char*> m_szOutputNames;
+
+	std::shared_ptr<Ort::Session> m_session;
+    Ort::Env env_;
+    Ort::SessionOptions session_options;
+public:
+
+	CTTransformer();
+	void InitPunc(const std::string &punc_model, const std::string &punc_config, const std::string &token_file, int thread_num);
+	~CTTransformer();
+	vector<int>  Infer(vector<int32_t> input_data);
+	string AddPunc(const char* sz_input, std::string language="zh-cn");
+};
+} // namespace funasr

runtime/onnxruntime/src/e2e-vad.h

@@ -0,0 +1,785 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+ * Contributed by zhuzizyf(China Telecom).
+*/
+
+#pragma once 
+
+#include <utility>
+#include <vector>
+#include <string>
+#include <map>
+#include <cmath>
+#include <algorithm>
+#include <iostream>
+#include <numeric>
+#include <cassert>
+
+namespace funasr {
+enum class VadStateMachine {
+    kVadInStateStartPointNotDetected = 1,
+    kVadInStateInSpeechSegment = 2,
+    kVadInStateEndPointDetected = 3
+};
+
+enum class FrameState {
+    kFrameStateInvalid = -1,
+    kFrameStateSpeech = 1,
+    kFrameStateSil = 0
+};
+
+// final voice/unvoice state per frame
+enum class AudioChangeState {
+    kChangeStateSpeech2Speech = 0,
+    kChangeStateSpeech2Sil = 1,
+    kChangeStateSil2Sil = 2,
+    kChangeStateSil2Speech = 3,
+    kChangeStateNoBegin = 4,
+    kChangeStateInvalid = 5
+};
+
+enum class VadDetectMode {
+    kVadSingleUtteranceDetectMode = 0,
+    kVadMutipleUtteranceDetectMode = 1
+};
+
+class VADXOptions {
+public:
+    int sample_rate;
+    int detect_mode;
+    int snr_mode;
+    int max_end_silence_time;
+    int max_start_silence_time;
+    bool do_start_point_detection;
+    bool do_end_point_detection;
+    int window_size_ms;
+    int sil_to_speech_time_thres;
+    int speech_to_sil_time_thres;
+    float speech_2_noise_ratio;
+    int do_extend;
+    int lookback_time_start_point;
+    int lookahead_time_end_point;
+    int max_single_segment_time;
+    int nn_eval_block_size;
+    int dcd_block_size;
+    float snr_thres;
+    int noise_frame_num_used_for_snr;
+    float decibel_thres;
+    float speech_noise_thres;
+    float fe_prior_thres;
+    int silence_pdf_num;
+    std::vector<int> sil_pdf_ids;
+    float speech_noise_thresh_low;
+    float speech_noise_thresh_high;
+    bool output_frame_probs;
+    int frame_in_ms;
+    int frame_length_ms;
+
+    explicit VADXOptions(
+            int sr = 16000,
+            int dm = static_cast<int>(VadDetectMode::kVadMutipleUtteranceDetectMode),
+            int sm = 0,
+            int mset = 800,
+            int msst = 3000,
+            bool dspd = true,
+            bool depd = true,
+            int wsm = 200,
+            int ststh = 150,
+            int sttsh = 150,
+            float s2nr = 1.0,
+            int de = 1,
+            int lbtps = 200,
+            int latsp = 100,
+            int mss = 15000,
+            int nebs = 8,
+            int dbs = 4,
+            float st = -100.0,
+            int nfnus = 100,
+            float dt = -100.0,
+            float snt = 0.9,
+            float fept = 1e-4,
+            int spn = 1,
+            std::vector<int> spids = {0},
+            float sntl = -0.1,
+            float snth = 0.3,
+            bool ofp = false,
+            int fim = 10,
+            int flm = 25
+    ) :
+            sample_rate(sr),
+            detect_mode(dm),
+            snr_mode(sm),
+            max_end_silence_time(mset),
+            max_start_silence_time(msst),
+            do_start_point_detection(dspd),
+            do_end_point_detection(depd),
+            window_size_ms(wsm),
+            sil_to_speech_time_thres(ststh),
+            speech_to_sil_time_thres(sttsh),
+            speech_2_noise_ratio(s2nr),
+            do_extend(de),
+            lookback_time_start_point(lbtps),
+            lookahead_time_end_point(latsp),
+            max_single_segment_time(mss),
+            nn_eval_block_size(nebs),
+            dcd_block_size(dbs),
+            snr_thres(st),
+            noise_frame_num_used_for_snr(nfnus),
+            decibel_thres(dt),
+            speech_noise_thres(snt),
+            fe_prior_thres(fept),
+            silence_pdf_num(spn),
+            sil_pdf_ids(std::move(spids)),
+            speech_noise_thresh_low(sntl),
+            speech_noise_thresh_high(snth),
+            output_frame_probs(ofp),
+            frame_in_ms(fim),
+            frame_length_ms(flm) {}
+};
+
+class E2EVadSpeechBufWithDoa {
+public:
+    int start_ms;
+    int end_ms;
+    std::vector<float> buffer;
+    bool contain_seg_start_point;
+    bool contain_seg_end_point;
+    int doa;
+
+    E2EVadSpeechBufWithDoa() :
+            start_ms(0),
+            end_ms(0),
+            buffer(),
+            contain_seg_start_point(false),
+            contain_seg_end_point(false),
+            doa(0) {}
+
+    void Reset() {
+        start_ms = 0;
+        end_ms = 0;
+        buffer.clear();
+        contain_seg_start_point = false;
+        contain_seg_end_point = false;
+        doa = 0;
+    }
+};
+
+class E2EVadFrameProb {
+public:
+    double noise_prob;
+    double speech_prob;
+    double score;
+    int frame_id;
+    int frm_state;
+
+    E2EVadFrameProb() :
+            noise_prob(0.0),
+            speech_prob(0.0),
+            score(0.0),
+            frame_id(0),
+            frm_state(0) {}
+};
+
+class WindowDetector {
+public:
+    int window_size_ms;
+    int sil_to_speech_time;
+    int speech_to_sil_time;
+    int frame_size_ms;
+    int win_size_frame;
+    int win_sum;
+    std::vector<int> win_state;
+    int cur_win_pos;
+    FrameState pre_frame_state;
+    FrameState cur_frame_state;
+    int sil_to_speech_frmcnt_thres;
+    int speech_to_sil_frmcnt_thres;
+    int voice_last_frame_count;
+    int noise_last_frame_count;
+    int hydre_frame_count;
+
+    WindowDetector(int window_size_ms, int sil_to_speech_time, int speech_to_sil_time, int frame_size_ms) :
+            window_size_ms(window_size_ms),
+            sil_to_speech_time(sil_to_speech_time),
+            speech_to_sil_time(speech_to_sil_time),
+            frame_size_ms(frame_size_ms),
+            win_size_frame(window_size_ms / frame_size_ms),
+            win_sum(0),
+            win_state(std::vector<int>(win_size_frame, 0)),
+            cur_win_pos(0),
+            pre_frame_state(FrameState::kFrameStateSil),
+            cur_frame_state(FrameState::kFrameStateSil),
+            sil_to_speech_frmcnt_thres(sil_to_speech_time / frame_size_ms),
+            speech_to_sil_frmcnt_thres(speech_to_sil_time / frame_size_ms),
+            voice_last_frame_count(0),
+            noise_last_frame_count(0),
+            hydre_frame_count(0) {}
+
+    void Reset() {
+        cur_win_pos = 0;
+        win_sum = 0;
+        win_state = std::vector<int>(win_size_frame, 0);
+        pre_frame_state = FrameState::kFrameStateSil;
+        cur_frame_state = FrameState::kFrameStateSil;
+        voice_last_frame_count = 0;
+        noise_last_frame_count = 0;
+        hydre_frame_count = 0;
+    }
+
+    int GetWinSize() {
+        return win_size_frame;
+    }
+
+    AudioChangeState DetectOneFrame(FrameState frameState, int frame_count) {
+        int cur_frame_state = 0;
+        if (frameState == FrameState::kFrameStateSpeech) {
+            cur_frame_state = 1;
+        } else if (frameState == FrameState::kFrameStateSil) {
+            cur_frame_state = 0;
+        } else {
+            return AudioChangeState::kChangeStateInvalid;
+        }
+        win_sum -= win_state[cur_win_pos];
+        win_sum += cur_frame_state;
+        win_state[cur_win_pos] = cur_frame_state;
+        cur_win_pos = (cur_win_pos + 1) % win_size_frame;
+        if (pre_frame_state == FrameState::kFrameStateSil && win_sum >= sil_to_speech_frmcnt_thres) {
+            pre_frame_state = FrameState::kFrameStateSpeech;
+            return AudioChangeState::kChangeStateSil2Speech;
+        }
+        if (pre_frame_state == FrameState::kFrameStateSpeech && win_sum <= speech_to_sil_frmcnt_thres) {
+            pre_frame_state = FrameState::kFrameStateSil;
+            return AudioChangeState::kChangeStateSpeech2Sil;
+        }
+        if (pre_frame_state == FrameState::kFrameStateSil) {
+            return AudioChangeState::kChangeStateSil2Sil;
+        }
+        if (pre_frame_state == FrameState::kFrameStateSpeech) {
+            return AudioChangeState::kChangeStateSpeech2Speech;
+        }
+        return AudioChangeState::kChangeStateInvalid;
+    }
+
+    int FrameSizeMs() {
+        return frame_size_ms;
+    }
+};
+
+class E2EVadModel {
+public:
+    E2EVadModel() {
+        this->vad_opts = VADXOptions();
+//    this->windows_detector = WindowDetector(200,150,150,10);
+        // this->encoder = encoder;
+        // init variables
+        this->is_final = false;
+        this->data_buf_start_frame = 0;
+        this->frm_cnt = 0;
+        this->latest_confirmed_speech_frame = 0;
+        this->lastest_confirmed_silence_frame = -1;
+        this->continous_silence_frame_count = 0;
+        this->vad_state_machine = VadStateMachine::kVadInStateStartPointNotDetected;
+        this->confirmed_start_frame = -1;
+        this->confirmed_end_frame = -1;
+        this->number_end_time_detected = 0;
+        this->sil_frame = 0;
+        this->sil_pdf_ids = this->vad_opts.sil_pdf_ids;
+        this->noise_average_decibel = -100.0;
+        this->pre_end_silence_detected = false;
+        this->next_seg = true;
+//    this->output_data_buf = [];
+        this->output_data_buf_offset = 0;
+//    this->frame_probs = [];
+        this->max_end_sil_frame_cnt_thresh =
+                this->vad_opts.max_end_silence_time - this->vad_opts.speech_to_sil_time_thres;
+        this->speech_noise_thres = this->vad_opts.speech_noise_thres;
+        this->max_time_out = false;
+//    this->decibel = [];
+        this->ResetDetection();
+    }
+
+    std::vector<std::vector<int>>
+    operator()(const std::vector<std::vector<float>> &score, const std::vector<float> &waveform, bool is_final = false,
+               bool online = false, int max_end_sil = 800, int max_single_segment_time = 15000,
+               float speech_noise_thres = 0.8, int sample_rate = 16000) {
+        max_end_sil_frame_cnt_thresh = max_end_sil - vad_opts.speech_to_sil_time_thres;
+        this->waveform = waveform;
+        this->vad_opts.max_single_segment_time = max_single_segment_time;
+        this->speech_noise_thres = speech_noise_thres;
+        this->vad_opts.sample_rate = sample_rate;
+
+        ComputeDecibel();
+        ComputeScores(score);
+        if (!is_final) {
+            DetectCommonFrames();
+        } else {
+            DetectLastFrames();
+        }
+
+        std::vector<std::vector<int>> segment_batch;
+        if (output_data_buf.size() > 0) {
+            for (size_t i = output_data_buf_offset; i < output_data_buf.size(); i++) {
+              int start_ms;
+              int end_ms;
+              if (online) {
+
+                if (!output_data_buf[i].contain_seg_start_point) {
+                  continue;
+                }
+                if (!next_seg && !output_data_buf[i].contain_seg_end_point) {
+                  continue;
+                }
+                start_ms = next_seg ? output_data_buf[i].start_ms : -1;
+
+                if (output_data_buf[i].contain_seg_end_point) {
+                  end_ms = output_data_buf[i].end_ms;
+                  next_seg = true;
+                  output_data_buf_offset += 1;
+                } else {
+                  end_ms = -1;
+                  next_seg = false;
+                }
+              } else {
+                if (!is_final &&
+                    (!output_data_buf[i].contain_seg_start_point || !output_data_buf[i].contain_seg_end_point)) {
+                  continue;
+                }
+                start_ms = output_data_buf[i].start_ms;
+                end_ms = output_data_buf[i].end_ms;
+                output_data_buf_offset += 1;
+              }
+                std::vector<int> segment = {start_ms, end_ms};
+                segment_batch.push_back(segment);
+            }
+        }
+
+        if (is_final) {
+            AllResetDetection();
+        }
+        return segment_batch;
+    }
+
+private:
+    VADXOptions vad_opts;
+    WindowDetector windows_detector = WindowDetector(200, 150, 150, 10);
+    bool is_final;
+    int data_buf_start_frame;
+    int frm_cnt;
+    int latest_confirmed_speech_frame;
+    int lastest_confirmed_silence_frame;
+    int continous_silence_frame_count;
+    VadStateMachine vad_state_machine;
+    int confirmed_start_frame;
+    int confirmed_end_frame;
+    int number_end_time_detected;
+    int sil_frame;
+    std::vector<int> sil_pdf_ids;
+    float noise_average_decibel;
+    bool pre_end_silence_detected;
+    bool next_seg;
+    std::vector<E2EVadSpeechBufWithDoa> output_data_buf;
+    int output_data_buf_offset;
+    std::vector<E2EVadFrameProb> frame_probs;
+    int max_end_sil_frame_cnt_thresh;
+    float speech_noise_thres;
+    std::vector<std::vector<float>> scores;
+    int idx_pre_chunk = 0;
+    bool max_time_out;
+    std::vector<float> decibel;
+    int data_buf_size = 0;
+    int data_buf_all_size = 0;
+    std::vector<float> waveform;
+
+    void AllResetDetection() {
+        is_final = false;
+        data_buf_start_frame = 0;
+        frm_cnt = 0;
+        latest_confirmed_speech_frame = 0;
+        lastest_confirmed_silence_frame = -1;
+        continous_silence_frame_count = 0;
+        vad_state_machine = VadStateMachine::kVadInStateStartPointNotDetected;
+        confirmed_start_frame = -1;
+        confirmed_end_frame = -1;
+        number_end_time_detected = 0;
+        sil_frame = 0;
+        sil_pdf_ids = vad_opts.sil_pdf_ids;
+        noise_average_decibel = -100.0;
+        pre_end_silence_detected = false;
+        next_seg = true;
+        output_data_buf.clear();
+        output_data_buf_offset = 0;
+        frame_probs.clear();
+        max_end_sil_frame_cnt_thresh = vad_opts.max_end_silence_time - vad_opts.speech_to_sil_time_thres;
+        speech_noise_thres = vad_opts.speech_noise_thres;
+        scores.clear();
+        idx_pre_chunk = 0;
+        max_time_out = false;
+        decibel.clear();
+        int data_buf_size = 0;
+        int data_buf_all_size = 0;
+        waveform.clear();
+        ResetDetection();
+    }
+
+    void ResetDetection() {
+        continous_silence_frame_count = 0;
+        latest_confirmed_speech_frame = 0;
+        lastest_confirmed_silence_frame = -1;
+        confirmed_start_frame = -1;
+        confirmed_end_frame = -1;
+        vad_state_machine = VadStateMachine::kVadInStateStartPointNotDetected;
+        windows_detector.Reset();
+        sil_frame = 0;
+        frame_probs.clear();
+    }
+
+    void ComputeDecibel() {
+        int frame_sample_length = int(vad_opts.frame_length_ms * vad_opts.sample_rate / 1000);
+        int frame_shift_length = int(vad_opts.frame_in_ms * vad_opts.sample_rate / 1000);
+        if (data_buf_all_size == 0) {
+          data_buf_all_size = waveform.size();
+          data_buf_size = data_buf_all_size;
+        } else {
+          data_buf_all_size += waveform.size();
+        }
+        for (int offset = 0; offset + frame_sample_length -1 < waveform.size(); offset += frame_shift_length) {
+            float sum = 0.0;
+            for (int i = 0; i < frame_sample_length; i++) {
+                sum += waveform[offset + i] * waveform[offset + i];
+            }
+            this->decibel.push_back(10 * log10(sum + 0.000001));
+        }
+    }
+
+    void ComputeScores(const std::vector<std::vector<float>> &scores) {
+        vad_opts.nn_eval_block_size = scores.size();
+        frm_cnt += scores.size();
+        this->scores = scores;
+    }
+
+    void PopDataBufTillFrame(int frame_idx) {
+      int frame_sample_length = int(vad_opts.frame_in_ms * vad_opts.sample_rate / 1000);
+      while (data_buf_start_frame < frame_idx) {
+        if (data_buf_size >= frame_sample_length) {
+          data_buf_start_frame += 1;
+          data_buf_size = data_buf_all_size - data_buf_start_frame * frame_sample_length;
+        }
+      }
+    }
+
+    void PopDataToOutputBuf(int start_frm, int frm_cnt, bool first_frm_is_start_point, bool last_frm_is_end_point,
+                            bool end_point_is_sent_end) {
+        PopDataBufTillFrame(start_frm);
+        int expected_sample_number = int(frm_cnt * vad_opts.sample_rate * vad_opts.frame_in_ms / 1000);
+        if (last_frm_is_end_point) {
+            int extra_sample = std::max(0, int(vad_opts.frame_length_ms * vad_opts.sample_rate / 1000 -
+                                               vad_opts.sample_rate * vad_opts.frame_in_ms / 1000));
+            expected_sample_number += int(extra_sample);
+        }
+        if (end_point_is_sent_end) {
+            expected_sample_number = std::max(expected_sample_number, data_buf_size);
+        }
+        if (data_buf_size < expected_sample_number) {
+            std::cout << "error in calling pop data_buf\n";
+        }
+        if (output_data_buf.size() == 0 || first_frm_is_start_point) {
+            output_data_buf.push_back(E2EVadSpeechBufWithDoa());
+            output_data_buf[output_data_buf.size() - 1].Reset();
+            output_data_buf[output_data_buf.size() - 1].start_ms = start_frm * vad_opts.frame_in_ms;
+            output_data_buf[output_data_buf.size() - 1].end_ms = output_data_buf[output_data_buf.size() - 1].start_ms;
+            output_data_buf[output_data_buf.size() - 1].doa = 0;
+        }
+        E2EVadSpeechBufWithDoa &cur_seg = output_data_buf.back();
+        if (cur_seg.end_ms != start_frm * vad_opts.frame_in_ms) {
+            std::cout << "warning\n";
+        }
+
+        int data_to_pop;
+        if (end_point_is_sent_end) {
+            data_to_pop = expected_sample_number;
+        } else {
+            data_to_pop = int(frm_cnt * vad_opts.frame_in_ms * vad_opts.sample_rate / 1000);
+        }
+        if (data_to_pop > data_buf_size) {
+            std::cout << "VAD data_to_pop is bigger than data_buf.size()!!!\n";
+            data_to_pop = data_buf_size;
+            expected_sample_number = data_buf_size;
+        }
+        cur_seg.doa = 0;
+        
+        if (cur_seg.end_ms != start_frm * vad_opts.frame_in_ms) {
+            std::cout << "Something wrong with the VAD algorithm\n";
+        }
+        data_buf_start_frame += frm_cnt;
+        cur_seg.end_ms = (start_frm + frm_cnt) * vad_opts.frame_in_ms;
+        if (first_frm_is_start_point) {
+            cur_seg.contain_seg_start_point = true;
+        }
+        if (last_frm_is_end_point) {
+            cur_seg.contain_seg_end_point = true;
+        }
+    }
+
+    void OnSilenceDetected(int valid_frame) {
+        lastest_confirmed_silence_frame = valid_frame;
+        if (vad_state_machine == VadStateMachine::kVadInStateStartPointNotDetected) {
+            PopDataBufTillFrame(valid_frame);
+        }
+        // silence_detected_callback_
+        // pass
+    }
+
+    void OnVoiceDetected(int valid_frame) {
+        latest_confirmed_speech_frame = valid_frame;
+        PopDataToOutputBuf(valid_frame, 1, false, false, false);
+    }
+
+    void OnVoiceStart(int start_frame, bool fake_result = false) {
+        if (vad_opts.do_start_point_detection) {
+            // pass
+        }
+        if (confirmed_start_frame != -1) {
+            std::cout << "not reset vad properly\n";
+        } else {
+            confirmed_start_frame = start_frame;
+        }
+        if (!fake_result && vad_state_machine == VadStateMachine::kVadInStateStartPointNotDetected) {
+            PopDataToOutputBuf(confirmed_start_frame, 1, true, false, false);
+        }
+    }
+
+
+    void OnVoiceEnd(int end_frame, bool fake_result, bool is_last_frame) {
+        for (int t = latest_confirmed_speech_frame + 1; t < end_frame; t++) {
+            OnVoiceDetected(t);
+        }
+        if (vad_opts.do_end_point_detection) {
+            // pass
+        }
+        if (confirmed_end_frame != -1) {
+            std::cout << "not reset vad properly\n";
+        } else {
+            confirmed_end_frame = end_frame;
+        }
+        if (!fake_result) {
+            sil_frame = 0;
+            PopDataToOutputBuf(confirmed_end_frame, 1, false, true, is_last_frame);
+        }
+        number_end_time_detected++;
+    }
+
+    void MaybeOnVoiceEndIfLastFrame(bool is_final_frame, int cur_frm_idx) {
+        if (is_final_frame) {
+            OnVoiceEnd(cur_frm_idx, false, true);
+            vad_state_machine = VadStateMachine::kVadInStateEndPointDetected;
+        }
+    }
+
+    int GetLatency() {
+        return int(LatencyFrmNumAtStartPoint() * vad_opts.frame_in_ms);
+    }
+
+    int LatencyFrmNumAtStartPoint() {
+        int vad_latency = windows_detector.GetWinSize();
+        if (vad_opts.do_extend) {
+            vad_latency += int(vad_opts.lookback_time_start_point / vad_opts.frame_in_ms);
+        }
+        return vad_latency;
+    }
+
+    FrameState GetFrameState(int t) {
+        FrameState frame_state = FrameState::kFrameStateInvalid;
+        float cur_decibel = decibel[t];
+        float cur_snr = cur_decibel - noise_average_decibel;
+        if (cur_decibel < vad_opts.decibel_thres) {
+            frame_state = FrameState::kFrameStateSil;
+            DetectOneFrame(frame_state, t, false);
+            return frame_state;
+        }
+        float sum_score = 0.0;
+        float noise_prob = 0.0;
+        assert(sil_pdf_ids.size() == vad_opts.silence_pdf_num);
+        if (sil_pdf_ids.size() > 0) {
+            std::vector<float> sil_pdf_scores;
+            for (auto sil_pdf_id: sil_pdf_ids) {
+                sil_pdf_scores.push_back(scores[t - idx_pre_chunk][sil_pdf_id]);
+            }
+            sum_score = accumulate(sil_pdf_scores.begin(), sil_pdf_scores.end(), 0.0);
+            noise_prob = log(sum_score) * vad_opts.speech_2_noise_ratio;
+            float total_score = 1.0;
+            sum_score = total_score - sum_score;
+        }
+        float speech_prob = log(sum_score);
+        if (vad_opts.output_frame_probs) {
+            E2EVadFrameProb frame_prob;
+            frame_prob.noise_prob = noise_prob;
+            frame_prob.speech_prob = speech_prob;
+            frame_prob.score = sum_score;
+            frame_prob.frame_id = t;
+            frame_probs.push_back(frame_prob);
+        }
+        if (exp(speech_prob) >= exp(noise_prob) + speech_noise_thres) {
+            if (cur_snr >= vad_opts.snr_thres && cur_decibel >= vad_opts.decibel_thres) {
+                frame_state = FrameState::kFrameStateSpeech;
+            } else {
+                frame_state = FrameState::kFrameStateSil;
+            }
+        } else {
+            frame_state = FrameState::kFrameStateSil;
+            if (noise_average_decibel < -99.9) {
+                noise_average_decibel = cur_decibel;
+            } else {
+                noise_average_decibel =
+                        (cur_decibel + noise_average_decibel * (vad_opts.noise_frame_num_used_for_snr - 1)) /
+                        vad_opts.noise_frame_num_used_for_snr;
+            }
+        }
+        return frame_state;
+    }
+
+    int DetectCommonFrames() {
+        if (vad_state_machine == VadStateMachine::kVadInStateEndPointDetected) {
+            return 0;
+        }
+        for (int i = vad_opts.nn_eval_block_size - 1; i >= 0; i--) {
+            FrameState frame_state = FrameState::kFrameStateInvalid;
+            frame_state = GetFrameState(frm_cnt - 1 - i);
+            DetectOneFrame(frame_state, frm_cnt - 1 - i, false);
+        }
+        idx_pre_chunk += scores.size();
+        return 0;
+    }
+
+    int DetectLastFrames() {
+        if (vad_state_machine == VadStateMachine::kVadInStateEndPointDetected) {
+            return 0;
+        }
+        for (int i = vad_opts.nn_eval_block_size - 1; i >= 0; i--) {
+            FrameState frame_state = FrameState::kFrameStateInvalid;
+            frame_state = GetFrameState(frm_cnt - 1 - i);
+            if (i != 0) {
+                DetectOneFrame(frame_state, frm_cnt - 1 - i, false);
+            } else {
+                DetectOneFrame(frame_state, frm_cnt - 1, true);
+            }
+        }
+        return 0;
+    }
+
+    void DetectOneFrame(FrameState cur_frm_state, int cur_frm_idx, bool is_final_frame) {
+        FrameState tmp_cur_frm_state = FrameState::kFrameStateInvalid;
+        if (cur_frm_state == FrameState::kFrameStateSpeech) {
+            if (std::fabs(1.0) > vad_opts.fe_prior_thres) {
+                tmp_cur_frm_state = FrameState::kFrameStateSpeech;
+            } else {
+                tmp_cur_frm_state = FrameState::kFrameStateSil;
+            }
+        } else if (cur_frm_state == FrameState::kFrameStateSil) {
+            tmp_cur_frm_state = FrameState::kFrameStateSil;
+        }
+        AudioChangeState state_change = windows_detector.DetectOneFrame(tmp_cur_frm_state, cur_frm_idx);
+        int frm_shift_in_ms = vad_opts.frame_in_ms;
+        if (AudioChangeState::kChangeStateSil2Speech == state_change) {
+            int silence_frame_count = continous_silence_frame_count;
+            continous_silence_frame_count = 0;
+            pre_end_silence_detected = false;
+            int start_frame = 0;
+            if (vad_state_machine == VadStateMachine::kVadInStateStartPointNotDetected) {
+                start_frame = std::max(data_buf_start_frame, cur_frm_idx - LatencyFrmNumAtStartPoint());
+                OnVoiceStart(start_frame);
+                vad_state_machine = VadStateMachine::kVadInStateInSpeechSegment;
+                for (int t = start_frame + 1; t <= cur_frm_idx; t++) {
+                    OnVoiceDetected(t);
+                }
+            } else if (vad_state_machine == VadStateMachine::kVadInStateInSpeechSegment) {
+                for (int t = latest_confirmed_speech_frame + 1; t < cur_frm_idx; t++) {
+                    OnVoiceDetected(t);
+                }
+                if (cur_frm_idx - confirmed_start_frame + 1 > vad_opts.max_single_segment_time / frm_shift_in_ms) {
+                    OnVoiceEnd(cur_frm_idx, false, false);
+                    vad_state_machine = VadStateMachine::kVadInStateEndPointDetected;
+                } else if (!is_final_frame) {
+                    OnVoiceDetected(cur_frm_idx);
+                } else {
+                    MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx);
+                }
+            }
+        } else if (AudioChangeState::kChangeStateSpeech2Sil == state_change) {
+            continous_silence_frame_count = 0;
+            if (vad_state_machine == VadStateMachine::kVadInStateStartPointNotDetected) {
+                // do nothing
+            } else if (vad_state_machine == VadStateMachine::kVadInStateInSpeechSegment) {
+                if (cur_frm_idx - confirmed_start_frame + 1 >
+                    vad_opts.max_single_segment_time / frm_shift_in_ms) {
+                    OnVoiceEnd(cur_frm_idx, false, false);
+                    vad_state_machine = VadStateMachine::kVadInStateEndPointDetected;
+                } else if (!is_final_frame) {
+                    OnVoiceDetected(cur_frm_idx);
+                } else {
+                    MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx);
+                }
+            }
+        } else if (AudioChangeState::kChangeStateSpeech2Speech == state_change) {
+            continous_silence_frame_count = 0;
+            if (vad_state_machine == VadStateMachine::kVadInStateInSpeechSegment) {
+                if (cur_frm_idx - confirmed_start_frame + 1 >
+                    vad_opts.max_single_segment_time / frm_shift_in_ms) {
+                    max_time_out = true;
+                    OnVoiceEnd(cur_frm_idx, false, false);
+                    vad_state_machine = VadStateMachine::kVadInStateEndPointDetected;
+                } else if (!is_final_frame) {
+                    OnVoiceDetected(cur_frm_idx);
+                } else {
+                    MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx);
+                }
+            }
+        } else if (AudioChangeState::kChangeStateSil2Sil == state_change) {
+            continous_silence_frame_count += 1;
+            if (vad_state_machine == VadStateMachine::kVadInStateStartPointNotDetected) {
+                if ((vad_opts.detect_mode == static_cast<int>(VadDetectMode::kVadSingleUtteranceDetectMode) &&
+                     (continous_silence_frame_count * frm_shift_in_ms > vad_opts.max_start_silence_time)) ||
+                    (is_final_frame && number_end_time_detected == 0)) {
+                    for (int t = lastest_confirmed_silence_frame + 1; t < cur_frm_idx; t++) {
+                        OnSilenceDetected(t);
+                    }
+                    OnVoiceStart(0, true);
+                    OnVoiceEnd(0, true, false);
+                    vad_state_machine = VadStateMachine::kVadInStateEndPointDetected;
+                } else {
+                    if (cur_frm_idx >= LatencyFrmNumAtStartPoint()) {
+                        OnSilenceDetected(cur_frm_idx - LatencyFrmNumAtStartPoint());
+                    }
+                }
+            } else if (vad_state_machine == VadStateMachine::kVadInStateInSpeechSegment) {
+                if (continous_silence_frame_count * frm_shift_in_ms >= max_end_sil_frame_cnt_thresh) {
+                    int lookback_frame = max_end_sil_frame_cnt_thresh / frm_shift_in_ms;
+                    if (vad_opts.do_extend) {
+                        lookback_frame -= vad_opts.lookahead_time_end_point / frm_shift_in_ms;
+                        lookback_frame -= 1;
+                        lookback_frame = std::max(0, lookback_frame);
+                    }
+                    OnVoiceEnd(cur_frm_idx - lookback_frame, false, false);
+                    vad_state_machine = VadStateMachine::kVadInStateEndPointDetected;
+                } else if (cur_frm_idx - confirmed_start_frame + 1 >
+                           vad_opts.max_single_segment_time / frm_shift_in_ms) {
+                    OnVoiceEnd(cur_frm_idx, false, false);
+                    vad_state_machine = VadStateMachine::kVadInStateEndPointDetected;
+                } else if (vad_opts.do_extend && !is_final_frame) {
+                    if (continous_silence_frame_count <= vad_opts.lookahead_time_end_point / frm_shift_in_ms) {
+                        OnVoiceDetected(cur_frm_idx);
+                    }
+                } else {
+                    MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx);
+                }
+            }
+        }
+        if (vad_state_machine == VadStateMachine::kVadInStateEndPointDetected &&
+            vad_opts.detect_mode == static_cast<int>(VadDetectMode::kVadMutipleUtteranceDetectMode)) {
+            ResetDetection();
+        }
+    }
+
+};
+
+} // namespace funasr

runtime/onnxruntime/src/encode_converter.cpp

@@ -0,0 +1,575 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+#include "encode_converter.h"
+#include <assert.h>
+
+
+namespace funasr {
+using namespace std;
+
+U16CHAR_T UTF16[8];
+U8CHAR_T UTF8[8];
+
+size_t MyUtf8ToUtf16(const U8CHAR_T* pu8, size_t ilen, U16CHAR_T* pu16);
+size_t MyUtf16ToUtf8(const U16CHAR_T* pu16, U8CHAR_T* pu8);
+
+
+void EncodeConverter::SwapEndian(U16CHAR_T* pbuf, size_t len)
+{
+  for (size_t i = 0; i < len; i++) {
+    pbuf[i] = ((pbuf[i] >> 8) | (pbuf[i] << 8));
+  }
+}
+
+
+size_t MyUtf16ToUtf8(const U16CHAR_T* pu16, U8CHAR_T* pu8)
+{
+  size_t n = 0;
+  if (pu16[0] <= 0x007F)
+  {
+    pu8[0] = (pu16[0] & 0x7F);
+    n = 1;
+  }
+  else if (pu16[0] >= 0x0080 &&  pu16[0] <= 0x07FF)
+  {
+    pu8[1] = (0x80 | (pu16[0] & 0x003F));
+    pu8[0] = (0xC0 | ((pu16[0] >> 6) & 0x001F));
+    n = 2;
+  }
+  else if (pu16[0] >= 0x0800)
+  {
+    pu8[2] = (0x80 | (pu16[0] & 0x003F));
+    pu8[1] = (0x80 | ((pu16[0] >> 6) & 0x003F));
+    pu8[0] = (0xE0 | ((pu16[0] >> 12) & 0x000F));
+    n = 3;
+  }
+
+  return n;
+}
+
+#define is2ByteUtf16(u16) ( (u16) >= 0x0080 && (u16) <= 0x07FF )
+#define is3ByteUtf16(u16) ( (u16) >= 0x0800 )
+
+size_t EncodeConverter::Utf16ToUtf8(const U16CHAR_T* pu16, U8CHAR_T* pu8)
+{
+  size_t n = 0;
+  if (pu16[0] <= 0x007F)
+  {
+    pu8[0] = (pu16[0] & 0x7F);
+    n = 1;
+  }
+  else if (pu16[0] >= 0x0080 &&  pu16[0] <= 0x07FF)
+  {
+    pu8[1] = (0x80 | (pu16[0] & 0x003F));
+    pu8[0] = (0xC0 | ((pu16[0] >> 6) & 0x001F));
+    n = 2;
+  }
+  else if (pu16[0] >= 0x0800)
+  {
+    pu8[2] = (0x80 | (pu16[0] & 0x003F));
+    pu8[1] = (0x80 | ((pu16[0] >> 6) & 0x003F));
+    pu8[0] = (0xE0 | ((pu16[0] >> 12) & 0x000F));
+    n = 3;
+  }
+
+  return n;
+}
+
+size_t EncodeConverter::Utf16ToUtf8(const U16CHAR_T* pu16, size_t ilen,
+    U8CHAR_T* pu8, size_t olen)
+{
+  size_t offset = 0;
+  size_t sz = 0;
+  /*
+  for (size_t i = 0; i < ilen && offset < static_cast<int>(olen) - 3; i++) {
+    sz = utf16ToUtf8(pu16 + i, pu8 + offset);
+    offset += sz;
+  }
+  */
+  for (size_t i = 0; i < ilen && static_cast<int>(offset) < static_cast<int>(olen); i++) {
+    sz = Utf16ToUtf8(pu16 + i, pu8 + offset);
+    if (static_cast<int>(offset + static_cast<int>(sz)) <= static_cast<int>(olen))
+        offset += sz;
+  }
+  
+ // pu8[offset] = '\0';
+  return offset;
+}
+
+u8string EncodeConverter::Utf16ToUtf8(const u16string& u16str)
+{
+  size_t buflen = u16str.length()*3 + 1;
+  U8CHAR_T* pu8 = new U8CHAR_T[buflen];
+  size_t len = Utf16ToUtf8(u16str.data(), u16str.length(),
+    pu8, buflen);
+  u8string u8str(pu8, len);
+  delete [] pu8;
+
+  return u8str;
+}
+
+size_t EncodeConverter::Utf8ToUtf16(const U8CHAR_T* pu8, U16CHAR_T* pu16)
+{
+  size_t n = 0;
+  if ((pu8[0] & 0xF0) == 0xE0)
+  {
+    if ((pu8[1] & 0xC0) == 0x80 &&
+        (pu8[2] & 0xC0) == 0x80)
+    {
+      pu16[0] = (((pu8[0] & 0x0F) << 4) | ((pu8[1] & 0x3C) >> 2));
+      pu16[0] <<= 8;
+      pu16[0] |= (((pu8[1] & 0x03) << 6) | (pu8[2] & 0x3F));
+    }
+    else
+    {
+      pu16[0] = defUniChar;
+    }
+    n = 3;
+  } 
+  else if ((pu8[0] & 0xE0) == 0xC0)
+  {
+    if ((pu8[1] & 0xC0) == 0x80) 
+    {
+      pu16[0] = ((pu8[0] & 0x1C) >> 2);
+      pu16[0] <<= 8;
+      pu16[0] |= (((pu8[0] & 0x03) << 6) | (pu8[1] & 0x3F));
+    }
+    else
+    {
+      pu16[0] = defUniChar;
+    }
+    n = 2;
+  } 
+  else if ((pu8[0] & 0x80) == 0x00) 
+  {
+    pu16[0] = pu8[0];
+    n = 1;
+  }
+
+  return n;
+}
+
+size_t MyUtf8ToUtf16(const U8CHAR_T* pu8, size_t ilen, U16CHAR_T* pu16)
+{
+  size_t n = 0;
+  if ((pu8[0] & 0xF0) == 0xE0 && ilen >= 3)
+  {
+    if ((pu8[1] & 0xC0) == 0x80 &&
+        (pu8[2] & 0xC0) == 0x80)
+    {
+      pu16[0] = (((pu8[0] & 0x0F) << 4) | ((pu8[1] & 0x3C) >> 2));
+      pu16[0] <<= 8;
+      pu16[0] |= (((pu8[1] & 0x03) << 6) | (pu8[2] & 0x3F));
+      n = 3;
+    }
+    else
+    {
+      pu16[0] = 0x0000;
+      n = 1;
+    }
+  } 
+  else if ((pu8[0] & 0xE0) == 0xC0 && ilen >= 2)
+  {
+    if ((pu8[1] & 0xC0) == 0x80) 
+    {
+      pu16[0] = ((pu8[0] & 0x1C) >> 2);
+      pu16[0] <<= 8;
+      pu16[0] |= (((pu8[0] & 0x03) << 6) | (pu8[1] & 0x3F));
+      n = 2;
+    }
+    else
+    {
+      pu16[0] = 0x0000;
+      n = 1;
+    }
+  } 
+  else if ((pu8[0] & 0x80) == 0x00) 
+  {
+    pu16[0] = pu8[0];
+    n = 1;
+  }
+  else
+  {
+      pu16[0] = 0x0000;
+      n = 1;
+  }
+  return n;
+}
+
+size_t EncodeConverter::Utf8ToUtf16(const U8CHAR_T* pu8, size_t ilen, U16CHAR_T* pu16)
+{
+  size_t n = 0;
+  if ((pu8[0] & 0xF0) == 0xE0 && ilen >= 3)
+  {
+    if ((pu8[1] & 0xC0) == 0x80 &&
+        (pu8[2] & 0xC0) == 0x80)
+    {
+      pu16[0] = (((pu8[0] & 0x0F) << 4) | ((pu8[1] & 0x3C) >> 2));
+      pu16[0] <<= 8;
+      pu16[0] |= (((pu8[1] & 0x03) << 6) | (pu8[2] & 0x3F));
+      n = 3;
+      if( !is3ByteUtf16(pu16[0]) )
+      {
+          pu16[0] = 0x0000;
+          n = 1;
+      }
+    }
+    else
+    {
+      pu16[0] = 0x0000;
+      n = 1;
+    }
+  } 
+  else if ((pu8[0] & 0xE0) == 0xC0 && ilen >= 2)
+  {
+    if ((pu8[1] & 0xC0) == 0x80) 
+    {
+      pu16[0] = ((pu8[0] & 0x1C) >> 2);
+      pu16[0] <<= 8;
+      pu16[0] |= (((pu8[0] & 0x03) << 6) | (pu8[1] & 0x3F));
+      n = 2;
+      if( !is2ByteUtf16(pu16[0]) )
+      {
+          pu16[0] = 0x0000;
+          n = 1;
+      }
+    }
+    else
+    {
+      pu16[0] = 0x0000;
+      n = 1;
+    }
+  } 
+  else if ((pu8[0] & 0x80) == 0x00) 
+  {
+    pu16[0] = pu8[0];
+    n = 1;
+  }
+  else
+  {
+      pu16[0] = 0x0000;
+      n = 1;
+  }
+
+  return n;
+  /*
+  size_t n = 0;
+  if ((pu8[0] & 0xF0) == 0xE0)
+  {
+    if (ilen >= 3 && (pu8[1] & 0xC0) == 0x80 &&
+        (pu8[2] & 0xC0) == 0x80)
+    {
+      pu16[0] = (((pu8[0] & 0x0F) << 4) | ((pu8[1] & 0x3C) >> 2));
+      pu16[0] <<= 8;
+      pu16[0] |= (((pu8[1] & 0x03) << 6) | (pu8[2] & 0x3F));
+    }
+    else
+    {
+      pu16[0] = defUniChar;
+    }
+    n = 3;
+  } 
+  else if ((pu8[0] & 0xE0) == 0xC0)
+  {
+    if( ilen >= 2 && (pu8[1] & 0xC0) == 0x80) 
+    {
+      pu16[0] = ((pu8[0] & 0x1C) >> 2);
+      pu16[0] <<= 8;
+      pu16[0] |= (((pu8[0] & 0x03) << 6) | (pu8[1] & 0x3F));
+    }
+    else
+    {
+      pu16[0] = defUniChar;
+    }
+    n = 2;
+  } 
+  else if ((pu8[0] & 0x80) == 0x00) 
+  {
+    pu16[0] = pu8[0];
+    n = 1;
+  }
+  else
+  {
+      pu16[0] = defUniChar;
+      n = 1;
+      for (size_t i = 1; i < ilen; i++)
+      {
+          if ((pu8[i] & 0xF0) == 0xE0 || (pu8[i] & 0xE0) == 0xC0 || (pu8[i] & 0x80) == 0x00)
+              break;
+          n++;
+      }
+  }
+
+  return n;
+  */
+}
+
+size_t EncodeConverter::Utf8ToUtf16(const U8CHAR_T* pu8, size_t ilen,
+    U16CHAR_T* pu16, size_t olen)
+{
+  int offset = 0;
+  size_t sz = 0;
+  for (size_t i = 0; i < ilen && offset < static_cast<int>(olen); offset ++)
+  {
+    sz = Utf8ToUtf16(pu8 + i, ilen - i, pu16 + offset);
+    i += sz;
+    if (sz == 0) {
+      // failed
+      // assert(sz != 0);
+      break;
+    }
+  }
+//  pu16[offset] = '\0';
+
+  return offset;
+}
+
+u16string EncodeConverter::Utf8ToUtf16(const u8string& u8str)
+{
+  U16CHAR_T* p16 = new U16CHAR_T[u8str.length() + 1];
+  size_t len = Utf8ToUtf16(u8str.data(), u8str.length(), 
+      p16, u8str.length() + 1);
+  u16string u16str(p16, len);
+  delete[] p16;
+
+  return u16str;
+}
+
+bool EncodeConverter::IsUTF8(const U8CHAR_T* pu8, size_t ilen)
+{
+  size_t i;
+  size_t n = 0;
+  for (i = 0; i < ilen; i += n)
+  {
+    if ((pu8[i] & 0xF0) == 0xE0 &&
+        (pu8[i + 1] & 0xC0) == 0x80 &&
+        (pu8[i + 2] & 0xC0) == 0x80)
+    {
+      n = 3;
+    }
+    else if ((pu8[i] & 0xE0) == 0xC0 &&
+        (pu8[i + 1] & 0xC0) == 0x80)
+    {
+      n = 2;
+    }
+    else if ((pu8[i] & 0x80) == 0x00)
+    {
+      n = 1;
+    }
+    else
+    {
+      break;
+    }
+  }
+
+  return i == ilen;
+}
+
+bool EncodeConverter::IsUTF8(const u8string& u8str)
+{
+  return IsUTF8(u8str.data(), u8str.length());
+}
+  
+size_t EncodeConverter::GetUTF8Len(const U8CHAR_T* pu8, size_t ilen)
+{
+  size_t i;
+  size_t n = 0;
+  size_t rlen = 0;
+  for (i = 0; i < ilen; i += n, rlen ++)
+  {
+    if ((pu8[i] & 0xF0) == 0xE0 &&
+        (pu8[i + 1] & 0xC0) == 0x80 &&
+        (pu8[i + 2] & 0xC0) == 0x80)
+    {
+      n = 3;
+    }
+    else if ((pu8[i] & 0xE0) == 0xC0 &&
+        (pu8[i + 1] & 0xC0) == 0x80)
+    {
+      n = 2;
+    }
+    else if ((pu8[i] & 0x80) == 0x00)
+    {
+      n = 1;
+    }
+    else
+    {
+      break;
+    }
+  }
+
+  if (i == ilen)
+    return 0;
+  else
+    return rlen;
+}
+
+size_t EncodeConverter::GetUTF8Len(const u8string& u8str)
+{
+  return GetUTF8Len(u8str.data(), u8str.length());
+}
+
+
+size_t EncodeConverter::Utf16ToUtf8Len(const U16CHAR_T* pu16, size_t ilen)
+{
+  int offset = 0;
+  for (size_t i = 0; i < ilen ; i++) {
+      if (pu16[i] <= 0x007F)
+      {
+        offset += 1;
+      }
+      else if (pu16[i] >= 0x0080 &&  pu16[i] <= 0x07FF)
+      {
+        offset += 2;
+      }
+      else if (pu16[i] >= 0x0800)
+      {
+        offset += 3;
+      }
+  }
+  
+  return offset;
+}
+
+uint16_t EncodeConverter::ToUni(const char* sc, int &len)
+{
+    uint16_t wide[2];
+    len = (int)Utf8ToUtf16((const U8CHAR_T*)sc, wide);
+    return wide[0];
+}
+
+bool EncodeConverter::IsAllChineseCharactor(const U8CHAR_T* pu8, size_t ilen) {
+    if (pu8 == nullptr || ilen <= 0) {
+        return false;
+    }
+
+    U16CHAR_T* p16 = new U16CHAR_T[ilen + 1];
+    size_t len = Utf8ToUtf16(pu8, ilen, p16, ilen + 1);
+    for (size_t i = 0; i < len; i++) {
+        if (p16[i] < 0x4e00 || p16[i] > 0x9fff) {
+            delete[] p16;
+            return false;
+        }
+    }
+    delete[] p16;
+    return true;
+}
+
+bool EncodeConverter::HasAlpha(const U8CHAR_T* pu8, size_t ilen) {
+  if (pu8 == nullptr || ilen <= 0) {
+    return false;
+  }
+  for (size_t i = 0; i < ilen; i++) {
+    if (pu8[i]> 0 && isalpha(pu8[i])){
+      return true;
+    }
+  }
+  return false;
+}
+
+
+bool EncodeConverter::IsAllAlpha(const U8CHAR_T* pu8, size_t ilen) {
+  if (pu8 == nullptr || ilen <= 0) {
+    return false;
+  }
+  for (size_t i = 0; i < ilen; i++) {
+    if (!(pu8[i]> 0 && isalpha(pu8[i]))){
+      return false;
+    }
+  }
+  return true;
+}
+
+bool EncodeConverter::IsAllAlphaAndPunct(const U8CHAR_T* pu8, size_t ilen) {
+  if (pu8 == nullptr || ilen <= 0) {
+    return false;
+  }
+  bool flag1 = HasAlpha(pu8, ilen);
+  if (flag1 == false) {
+    return false;
+  }
+
+  for (size_t i = 0; i < ilen; i++) {
+    if (!(pu8[i]> 0 && (isalpha(pu8[i]) || (ispunct(pu8[i]))))){
+      return false;
+    }
+  }
+  return true;
+}
+
+bool EncodeConverter::IsAllAlphaAndDigit(const U8CHAR_T* pu8, size_t ilen) {
+  if (pu8 == nullptr || ilen <= 0) {
+    return false;
+  }
+  bool flag1 = HasAlpha(pu8, ilen);
+  if (flag1 == false) {
+    return false;
+  }
+
+  for (size_t i = 0; i < ilen; i++) {
+    if (!(pu8[i]> 0 && (isalnum(pu8[i]) || isalpha(pu8[i]) || pu8[i] == '\''))){
+      return false;
+    }
+  }
+  return true;
+}
+bool EncodeConverter::IsAllAlphaAndDigitAndBlank(const U8CHAR_T* pu8, size_t ilen) {
+  if (pu8 == nullptr || ilen <= 0) {
+    return false;
+  }
+  for (size_t i = 0; i < ilen; i++) {
+    if (!(pu8[i]> 0 && (isalnum(pu8[i]) || isalpha(pu8[i]) || isblank(pu8[i]) || pu8[i] == '\''))){
+      return false;
+    }
+  }
+  return true;
+}
+bool EncodeConverter::NeedAddTailBlank(std::string str) {
+  U8CHAR_T *pu8 = (U8CHAR_T*)str.data();
+  size_t ilen = str.size();
+  if (pu8 == nullptr || ilen <= 0) {
+    return false;
+  }
+  if (IsAllAlpha(pu8, ilen) || IsAllAlphaAndPunct(pu8, ilen) || IsAllAlphaAndDigit(pu8, ilen)) {
+    return true;
+  } else {
+    return false;
+  }
+}
+std::vector<std::string> EncodeConverter::MergeEnglishWord(std::vector<std::string> &str_vec_input,
+                                                           std::vector<int> &merge_mask) {
+  std::vector<std::string> output;
+  for (int i = 0; i < merge_mask.size(); i++) {
+    if (merge_mask[i] == 1 && i > 0) {
+      output[output.size() - 1] += str_vec_input[i];
+    } else {
+      output.push_back(str_vec_input[i]);
+    }
+  }
+  str_vec_input.swap(output);
+  return str_vec_input;
+}
+size_t EncodeConverter::Utf8ToCharset(const std::string &input, std::vector<std::string> &output) {
+  std::string ch; 
+  for (size_t i = 0, len = 0; i != input.length(); i += len) {
+    unsigned char byte = (unsigned)input[i];
+    if (byte >= 0xFC) // lenght 6
+      len = 6;  
+    else if (byte >= 0xF8)
+      len = 5;
+    else if (byte >= 0xF0)
+      len = 4;
+    else if (byte >= 0xE0)
+      len = 3;
+    else if (byte >= 0xC0)
+      len = 2;
+    else
+      len = 1;
+    ch = input.substr(i, len);
+    output.push_back(ch);
+  }   
+  return output.size();
+}
+}

runtime/onnxruntime/src/encode_converter.h

@@ -0,0 +1,109 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+#ifndef __WS__ENCODE_CONVERTER_H__
+#define __WS__ENCODE_CONVERTER_H__
+
+#include <string>
+#include <stdint.h>
+#include <vector>
+#ifdef _MSC_VER
+#include <windows.h>
+#endif // _MSC_VER
+
+namespace funasr {
+    typedef unsigned char           U8CHAR_T;
+    typedef unsigned short          U16CHAR_T;
+    typedef std::basic_string<U8CHAR_T>  u8string;
+    typedef std::basic_string<U16CHAR_T> u16string;
+
+    class EncodeConverter {
+    public:
+        static const U16CHAR_T defUniChar = 0x25a1;  //WHITE SQUARE
+
+    public:
+        static void SwapEndian(U16CHAR_T* pbuf, size_t len);
+
+        static size_t Utf16ToUtf8(const U16CHAR_T* pu16, U8CHAR_T* pu8);
+
+        ///< @param pu16 UTF16 string
+        ///< @param pu8 UTF8 string
+        static size_t Utf16ToUtf8(const U16CHAR_T* pu16, size_t ilen,
+                                  U8CHAR_T* pu8, size_t olen);
+
+        static u8string Utf16ToUtf8(const u16string& u16str);
+
+        static size_t Utf8ToUtf16(const U8CHAR_T* pu8, U16CHAR_T* pu16);
+
+        static size_t Utf8ToUtf16(const U8CHAR_T* pu8, size_t ilen, U16CHAR_T* pu16);
+
+        ///< @param pu8 UTF8 string
+        ///< @param pu16 UTF16 string
+        static size_t Utf8ToUtf16(const U8CHAR_T* pu8, size_t ilen,
+                                  U16CHAR_T* pu16, size_t olen);
+
+        static u16string Utf8ToUtf16(const u8string& u8str);
+
+        ///< @param pu8 string
+        ///< @return if string is encoded as UTF8 - true, otherwise false
+        static bool IsUTF8(const U8CHAR_T* pu8, size_t ilen);
+
+        ///< @param u8str string
+        ///< @return if string is encoded as UTF8 - true, otherwise false
+        static bool IsUTF8(const u8string& u8str);
+
+        ///< @param UTF8 string
+        ///< @return the word number of UTF8
+        static size_t GetUTF8Len(const U8CHAR_T* pu8, size_t ilen);
+
+        ///< @param UTF8 string
+        ///< @return the word number of UTF8
+        static size_t GetUTF8Len(const u8string& u8str);
+
+        ///< @param pu16 UTF16 string
+        ///< @param ilen UTF16 length
+        ///< @return UTF8 string length
+        static size_t Utf16ToUtf8Len(const U16CHAR_T* pu16, size_t ilen);
+
+        static uint16_t ToUni(const char* sc, int &len);
+
+        static bool IsChineseCharacter(U16CHAR_T &u16) {
+            return (u16 >= 0x4e00 && u16 <= 0x9fff)  // common
+                || (u16 >= 0x3400 && u16 <= 0x4dff); // rare, extension A
+        }
+
+        // whether the string is all Chinese
+        static bool IsAllChineseCharactor(const U8CHAR_T* pu8, size_t ilen);
+        static bool HasAlpha(const U8CHAR_T* pu8, size_t ilen);
+        static bool NeedAddTailBlank(std::string str);
+        static bool IsAllAlpha(const U8CHAR_T* pu8, size_t ilen);
+        static bool IsAllAlphaAndPunct(const U8CHAR_T* pu8, size_t ilen);
+        static bool IsAllAlphaAndDigit(const U8CHAR_T* pu8, size_t ilen);
+        static bool IsAllAlphaAndDigitAndBlank(const U8CHAR_T* pu8, size_t ilen);
+        static std::vector<std::string> MergeEnglishWord(std::vector<std::string> &str_vec_input,
+                                                         std::vector<int> &merge_mask);
+        static size_t Utf8ToCharset(const std::string &input, std::vector<std::string> &output);
+
+#ifdef _MSC_VER
+        // convert to the local ansi page
+        static std::string UTF8ToLocaleAnsi(const std::string& strUTF8) {
+            int len = MultiByteToWideChar(CP_UTF8, 0, strUTF8.c_str(), -1, nullptr, 0);
+            unsigned short*wszGBK = new unsigned short[len + 1];
+            memset(wszGBK, 0, len * 2 + 2);
+            MultiByteToWideChar(CP_UTF8, 0, (LPCCH)strUTF8.c_str(), -1, (LPWSTR)wszGBK, len);
+
+            len = WideCharToMultiByte(CP_ACP, 0, (LPCWCH)wszGBK, -1, nullptr, 0, nullptr, nullptr);
+            char *szGBK = new char[len + 1];
+            memset(szGBK, 0, len + 1);
+            WideCharToMultiByte(CP_ACP, 0, (LPCWCH)wszGBK, -1, szGBK, len, nullptr, nullptr);
+            std::string strTemp(szGBK);
+            delete[]szGBK;
+            delete[]wszGBK;
+            return strTemp;
+        }
+#endif
+    };
+}
+
+#endif //__WS_ENCODE_CONVERTER_H__

runtime/onnxruntime/src/fsmn-vad-online.cpp

@@ -0,0 +1,215 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#include <fstream>
+#include "precomp.h"
+
+namespace funasr {
+
+void FsmnVadOnline::FbankKaldi(float sample_rate, std::vector<std::vector<float>> &vad_feats,
+                               std::vector<float> &waves) {
+    knf::OnlineFbank fbank(fbank_opts_);
+    // cache merge
+    waves.insert(waves.begin(), input_cache_.begin(), input_cache_.end());
+    int frame_number = ComputeFrameNum(waves.size(), frame_sample_length_, frame_shift_sample_length_);
+    // Send the audio after the last frame shift position to the cache
+    input_cache_.clear();
+    input_cache_.insert(input_cache_.begin(), waves.begin() + frame_number * frame_shift_sample_length_, waves.end());
+    if (frame_number == 0) {
+        return;
+    }
+    // Delete audio that haven't undergone fbank processing
+    waves.erase(waves.begin() + (frame_number - 1) * frame_shift_sample_length_ + frame_sample_length_, waves.end());
+
+    std::vector<float> buf(waves.size());
+    for (int32_t i = 0; i != waves.size(); ++i) {
+        buf[i] = waves[i] * 32768;
+    }
+    fbank.AcceptWaveform(sample_rate, buf.data(), buf.size());
+    // fbank.AcceptWaveform(sample_rate, &waves[0], waves.size());
+    int32_t frames = fbank.NumFramesReady();
+    for (int32_t i = 0; i != frames; ++i) {
+        const float *frame = fbank.GetFrame(i);
+        vector<float> frame_vector(frame, frame + fbank_opts_.mel_opts.num_bins);
+        vad_feats.emplace_back(frame_vector);
+    }
+}
+
+void FsmnVadOnline::ExtractFeats(float sample_rate, vector<std::vector<float>> &vad_feats,
+                                 vector<float> &waves, bool input_finished) {
+  FbankKaldi(sample_rate, vad_feats, waves);
+  // cache deal & online lfr,cmvn
+  if (vad_feats.size() > 0) {
+    if (!reserve_waveforms_.empty()) {
+      waves.insert(waves.begin(), reserve_waveforms_.begin(), reserve_waveforms_.end());
+    }
+    if (lfr_splice_cache_.empty()) {
+      for (int i = 0; i < (lfr_m - 1) / 2; i++) {
+        lfr_splice_cache_.emplace_back(vad_feats[0]);
+      }
+    }
+    if (vad_feats.size() + lfr_splice_cache_.size() >= lfr_m) {
+      vad_feats.insert(vad_feats.begin(), lfr_splice_cache_.begin(), lfr_splice_cache_.end());
+      int frame_from_waves = (waves.size() - frame_sample_length_) / frame_shift_sample_length_ + 1;
+      int minus_frame = reserve_waveforms_.empty() ? (lfr_m - 1) / 2 : 0;
+      int lfr_splice_frame_idxs = OnlineLfrCmvn(vad_feats, input_finished);
+      int reserve_frame_idx = std::abs(lfr_splice_frame_idxs - minus_frame);
+      reserve_waveforms_.clear();
+      reserve_waveforms_.insert(reserve_waveforms_.begin(),
+                                waves.begin() + reserve_frame_idx * frame_shift_sample_length_,
+                                waves.begin() + frame_from_waves * frame_shift_sample_length_);
+      int sample_length = (frame_from_waves - 1) * frame_shift_sample_length_ + frame_sample_length_;
+      waves.erase(waves.begin() + sample_length, waves.end());
+    } else {
+      reserve_waveforms_.clear();
+      reserve_waveforms_.insert(reserve_waveforms_.begin(),
+                                waves.begin() + frame_sample_length_ - frame_shift_sample_length_, waves.end());
+      lfr_splice_cache_.insert(lfr_splice_cache_.end(), vad_feats.begin(), vad_feats.end());
+    }
+  } else {
+    if (input_finished) {
+      if (!reserve_waveforms_.empty()) {
+        waves = reserve_waveforms_;
+      }
+      vad_feats = lfr_splice_cache_;
+      if(vad_feats.size() == 0){
+        LOG(ERROR) << "vad_feats's size is 0";
+      }else{
+        OnlineLfrCmvn(vad_feats, input_finished);
+      }
+    }
+  }
+  if(input_finished){
+      Reset();
+      ResetCache();
+  }
+}
+
+int FsmnVadOnline::OnlineLfrCmvn(vector<vector<float>> &vad_feats, bool input_finished) {
+    vector<vector<float>> out_feats;
+    int T = vad_feats.size();
+    int T_lrf = ceil((T - (lfr_m - 1) / 2) / (float)lfr_n);
+    int lfr_splice_frame_idxs = T_lrf;
+    vector<float> p;
+    for (int i = 0; i < T_lrf; i++) {
+        if (lfr_m <= T - i * lfr_n) {
+            for (int j = 0; j < lfr_m; j++) {
+                p.insert(p.end(), vad_feats[i * lfr_n + j].begin(), vad_feats[i * lfr_n + j].end());
+            }
+            out_feats.emplace_back(p);
+            p.clear();
+        } else {
+            if (input_finished) {
+                int num_padding = lfr_m - (T - i * lfr_n);
+                for (int j = 0; j < (vad_feats.size() - i * lfr_n); j++) {
+                    p.insert(p.end(), vad_feats[i * lfr_n + j].begin(), vad_feats[i * lfr_n + j].end());
+                }
+                for (int j = 0; j < num_padding; j++) {
+                    p.insert(p.end(), vad_feats[vad_feats.size() - 1].begin(), vad_feats[vad_feats.size() - 1].end());
+                }
+                out_feats.emplace_back(p);
+                p.clear();
+            } else {
+                lfr_splice_frame_idxs = i;
+                break;
+            }
+        }
+    }
+    lfr_splice_frame_idxs = std::min(T - 1, lfr_splice_frame_idxs * lfr_n);
+    lfr_splice_cache_.clear();
+    lfr_splice_cache_.insert(lfr_splice_cache_.begin(), vad_feats.begin() + lfr_splice_frame_idxs, vad_feats.end());
+
+    // Apply cmvn
+    for (auto &out_feat: out_feats) {
+        for (int j = 0; j < means_list_.size(); j++) {
+            out_feat[j] = (out_feat[j] + means_list_[j]) * vars_list_[j];
+        }
+    }
+    vad_feats = out_feats;
+    return lfr_splice_frame_idxs;
+}
+
+std::vector<std::vector<int>>
+FsmnVadOnline::Infer(std::vector<float> &waves, bool input_finished) {
+    std::vector<std::vector<int>> vad_segments;
+    std::vector<std::vector<float>> vad_feats;
+    std::vector<std::vector<float>> vad_probs;
+    ExtractFeats(vad_sample_rate_, vad_feats, waves, input_finished);
+    if(vad_feats.size() == 0){
+      return vad_segments;
+    }
+    fsmnvad_handle_->Forward(vad_feats, &vad_probs, &in_cache_, input_finished);
+    if(vad_probs.size() == 0){
+      return vad_segments;
+    }
+
+    vad_segments = vad_scorer(vad_probs, waves, input_finished, true, vad_silence_duration_, vad_max_len_,
+                              vad_speech_noise_thres_, vad_sample_rate_);
+    return vad_segments;
+}
+
+void FsmnVadOnline::InitCache(){
+  std::vector<float> cache_feats(128 * 19 * 1, 0);
+  for (int i=0;i<4;i++){
+    in_cache_.emplace_back(cache_feats);
+  }
+};
+
+void FsmnVadOnline::Reset(){
+  in_cache_.clear();
+  InitCache();
+};
+
+void FsmnVadOnline::Test() {
+}
+
+void FsmnVadOnline::InitOnline(std::shared_ptr<Ort::Session> &vad_session,
+                               Ort::Env &env,
+                               std::vector<const char *> &vad_in_names,
+                               std::vector<const char *> &vad_out_names,
+                               knf::FbankOptions &fbank_opts,
+                               std::vector<float> &means_list,
+                               std::vector<float> &vars_list,
+                               int vad_sample_rate,
+                               int vad_silence_duration,
+                               int vad_max_len,
+                               double vad_speech_noise_thres) {
+    vad_session_ = vad_session;
+    vad_in_names_ = vad_in_names;
+    vad_out_names_ = vad_out_names;
+    fbank_opts_ = fbank_opts;
+    means_list_ = means_list;
+    vars_list_ = vars_list;
+    vad_sample_rate_ = vad_sample_rate;
+    vad_silence_duration_ = vad_silence_duration;
+    vad_max_len_ = vad_max_len;
+    vad_speech_noise_thres_ = vad_speech_noise_thres;
+
+    frame_sample_length_ = vad_sample_rate_ / 1000 * 25;;
+    frame_shift_sample_length_ = vad_sample_rate_ / 1000 * 10;
+
+    // 2pass
+    audio_handle = make_unique<Audio>(vad_sample_rate,1);
+}
+
+FsmnVadOnline::~FsmnVadOnline() {
+}
+
+FsmnVadOnline::FsmnVadOnline(FsmnVad* fsmnvad_handle):fsmnvad_handle_(std::move(fsmnvad_handle)),session_options_{}{
+   InitCache();
+   InitOnline(fsmnvad_handle_->vad_session_,
+              fsmnvad_handle_->env_,
+              fsmnvad_handle_->vad_in_names_,
+              fsmnvad_handle_->vad_out_names_,
+              fsmnvad_handle_->fbank_opts_,
+              fsmnvad_handle_->means_list_,
+              fsmnvad_handle_->vars_list_,
+              fsmnvad_handle_->vad_sample_rate_,
+              fsmnvad_handle_->vad_silence_duration_,
+              fsmnvad_handle_->vad_max_len_,
+              fsmnvad_handle_->vad_speech_noise_thres_);
+}
+
+} // namespace funasr

runtime/onnxruntime/src/fsmn-vad-online.h

@@ -0,0 +1,92 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#pragma once 
+#include "precomp.h"
+
+namespace funasr {
+class FsmnVadOnline : public VadModel {
+/**
+ * Author: Speech Lab of DAMO Academy, Alibaba Group
+ * Deep-FSMN for Large Vocabulary Continuous Speech Recognition
+ * https://arxiv.org/abs/1803.05030
+*/
+
+public:
+    explicit FsmnVadOnline(FsmnVad* fsmnvad_handle);
+    ~FsmnVadOnline();
+    void Test();
+    std::vector<std::vector<int>> Infer(std::vector<float> &waves, bool input_finished);
+    void ExtractFeats(float sample_rate, vector<vector<float>> &vad_feats, vector<float> &waves, bool input_finished);
+    void Reset();
+    int GetVadSampleRate() { return vad_sample_rate_; };
+
+    // 2pass
+    std::unique_ptr<Audio> audio_handle = nullptr;
+
+private:
+    E2EVadModel vad_scorer = E2EVadModel();
+    // std::unique_ptr<FsmnVad> fsmnvad_handle_;
+    FsmnVad* fsmnvad_handle_ = nullptr;
+
+    void FbankKaldi(float sample_rate, std::vector<std::vector<float>> &vad_feats,
+                    std::vector<float> &waves);
+    int OnlineLfrCmvn(vector<vector<float>> &vad_feats, bool input_finished);
+    void InitVad(const std::string &vad_model, const std::string &vad_cmvn, const std::string &vad_config, int thread_num){}
+    void InitCache();
+    void InitOnline(std::shared_ptr<Ort::Session> &vad_session,
+                    Ort::Env &env,
+                    std::vector<const char *> &vad_in_names,
+                    std::vector<const char *> &vad_out_names,
+                    knf::FbankOptions &fbank_opts,
+                    std::vector<float> &means_list,
+                    std::vector<float> &vars_list,
+                    int vad_sample_rate,
+                    int vad_silence_duration,
+                    int vad_max_len,
+                    double vad_speech_noise_thres);
+
+    static int ComputeFrameNum(int sample_length, int frame_sample_length, int frame_shift_sample_length) {
+        int frame_num = static_cast<int>((sample_length - frame_sample_length) / frame_shift_sample_length + 1);
+        if (frame_num >= 1 && sample_length >= frame_sample_length)
+            return frame_num;
+        else
+            return 0;
+    }
+    void ResetCache() {
+        reserve_waveforms_.clear();
+        input_cache_.clear();
+        lfr_splice_cache_.clear();
+    }
+
+    // from fsmnvad_handle_
+    std::shared_ptr<Ort::Session> vad_session_ = nullptr;
+    Ort::Env env_;
+    Ort::SessionOptions session_options_;
+    std::vector<const char *> vad_in_names_;
+    std::vector<const char *> vad_out_names_;
+    knf::FbankOptions fbank_opts_;
+    std::vector<float> means_list_;
+    std::vector<float> vars_list_;
+
+    std::vector<std::vector<float>> in_cache_;
+    // The reserved waveforms by fbank
+    std::vector<float> reserve_waveforms_;
+    // waveforms reserved after last shift position
+    std::vector<float> input_cache_;
+    // lfr reserved cache
+    std::vector<std::vector<float>> lfr_splice_cache_;
+
+    int vad_sample_rate_ = MODEL_SAMPLE_RATE;
+    int vad_silence_duration_ = VAD_SILENCE_DURATION;
+    int vad_max_len_ = VAD_MAX_LEN;
+    double vad_speech_noise_thres_ = VAD_SPEECH_NOISE_THRES;
+    int lfr_m = VAD_LFR_M;
+    int lfr_n = VAD_LFR_N;
+    int frame_sample_length_ = vad_sample_rate_ / 1000 * 25;;
+    int frame_shift_sample_length_ = vad_sample_rate_ / 1000 * 10;
+};
+
+} // namespace funasr

runtime/onnxruntime/src/fsmn-vad.cpp

@@ -0,0 +1,274 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#include <fstream>
+#include "precomp.h"
+
+namespace funasr {
+void FsmnVad::InitVad(const std::string &vad_model, const std::string &vad_cmvn, const std::string &vad_config, int thread_num) {
+    session_options_.SetIntraOpNumThreads(thread_num);
+    session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
+    session_options_.DisableCpuMemArena();
+
+    ReadModel(vad_model.c_str());
+    LoadCmvn(vad_cmvn.c_str());
+    LoadConfigFromYaml(vad_config.c_str());
+    InitCache();
+}
+
+void FsmnVad::LoadConfigFromYaml(const char* filename){
+
+    YAML::Node config;
+    try{
+        config = YAML::LoadFile(filename);
+    }catch(exception const &e){
+        LOG(ERROR) << "Error loading file, yaml file error or not exist.";
+        exit(-1);
+    }
+
+    try{
+        YAML::Node frontend_conf = config["frontend_conf"];
+        YAML::Node post_conf = config["model_conf"];
+
+        this->vad_sample_rate_ = frontend_conf["fs"].as<int>();
+        this->vad_silence_duration_ =  post_conf["max_end_silence_time"].as<int>();
+        this->vad_max_len_ = post_conf["max_single_segment_time"].as<int>();
+        this->vad_speech_noise_thres_ = post_conf["speech_noise_thres"].as<double>();
+
+        fbank_opts_.frame_opts.dither = frontend_conf["dither"].as<float>();
+        fbank_opts_.mel_opts.num_bins = frontend_conf["n_mels"].as<int>();
+        fbank_opts_.frame_opts.samp_freq = (float)vad_sample_rate_;
+        fbank_opts_.frame_opts.window_type = frontend_conf["window"].as<string>();
+        fbank_opts_.frame_opts.frame_shift_ms = frontend_conf["frame_shift"].as<float>();
+        fbank_opts_.frame_opts.frame_length_ms = frontend_conf["frame_length"].as<float>();
+        fbank_opts_.energy_floor = 0;
+        fbank_opts_.mel_opts.debug_mel = false;
+    }catch(exception const &e){
+        LOG(ERROR) << "Error when load argument from vad config YAML.";
+        exit(-1);
+    }
+}
+
+void FsmnVad::ReadModel(const char* vad_model) {
+    try {
+        vad_session_ = std::make_shared<Ort::Session>(
+                env_, ORTCHAR(vad_model), session_options_);
+        LOG(INFO) << "Successfully load model from " << vad_model;
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load vad onnx model: " << e.what();
+        exit(-1);
+    }
+    GetInputNames(vad_session_.get(), m_strInputNames, vad_in_names_);
+    GetOutputNames(vad_session_.get(), m_strOutputNames, vad_out_names_);
+}
+
+void FsmnVad::Forward(
+        const std::vector<std::vector<float>> &chunk_feats,
+        std::vector<std::vector<float>> *out_prob,
+        std::vector<std::vector<float>> *in_cache,
+        bool is_final) {
+    Ort::MemoryInfo memory_info =
+            Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
+
+    int num_frames = chunk_feats.size();
+    const int feature_dim = chunk_feats[0].size();
+
+    //  2. Generate input nodes tensor
+    // vad node { batch,frame number,feature dim }
+    const int64_t vad_feats_shape[3] = {1, num_frames, feature_dim};
+    std::vector<float> vad_feats;
+    for (const auto &chunk_feat: chunk_feats) {
+        vad_feats.insert(vad_feats.end(), chunk_feat.begin(), chunk_feat.end());
+    }
+    Ort::Value vad_feats_ort = Ort::Value::CreateTensor<float>(
+            memory_info, vad_feats.data(), vad_feats.size(), vad_feats_shape, 3);
+    
+    // 3. Put nodes into onnx input vector
+    std::vector<Ort::Value> vad_inputs;
+    vad_inputs.emplace_back(std::move(vad_feats_ort));
+    // 4 caches
+    // cache node {batch,128,19,1}
+    const int64_t cache_feats_shape[4] = {1, 128, 19, 1};
+    for (int i = 0; i < in_cache->size(); i++) {
+      vad_inputs.emplace_back(std::move(Ort::Value::CreateTensor<float>(
+              memory_info, (*in_cache)[i].data(), (*in_cache)[i].size(), cache_feats_shape, 4)));
+    }
+  
+    // 4. Onnx infer
+    std::vector<Ort::Value> vad_ort_outputs;
+    try {
+        vad_ort_outputs = vad_session_->Run(
+                Ort::RunOptions{nullptr}, vad_in_names_.data(), vad_inputs.data(),
+                vad_inputs.size(), vad_out_names_.data(), vad_out_names_.size());
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when run vad onnx forword: " << (e.what());
+        return;
+    }
+
+    // 5. Change infer result to output shapes
+    float *logp_data = vad_ort_outputs[0].GetTensorMutableData<float>();
+    auto type_info = vad_ort_outputs[0].GetTensorTypeAndShapeInfo();
+
+    int num_outputs = type_info.GetShape()[1];
+    int output_dim = type_info.GetShape()[2];
+    out_prob->resize(num_outputs);
+    for (int i = 0; i < num_outputs; i++) {
+        (*out_prob)[i].resize(output_dim);
+        memcpy((*out_prob)[i].data(), logp_data + i * output_dim,
+               sizeof(float) * output_dim);
+    }
+  
+    // get 4 caches outputs,each size is 128*19
+    if(!is_final){
+        for (int i = 1; i < 5; i++) {
+        float* data = vad_ort_outputs[i].GetTensorMutableData<float>();
+        memcpy((*in_cache)[i-1].data(), data, sizeof(float) * 128*19);
+        }
+    }
+}
+
+void FsmnVad::FbankKaldi(float sample_rate, std::vector<std::vector<float>> &vad_feats,
+                         std::vector<float> &waves) {
+    knf::OnlineFbank fbank(fbank_opts_);
+
+    std::vector<float> buf(waves.size());
+    for (int32_t i = 0; i != waves.size(); ++i) {
+        buf[i] = waves[i] * 32768;
+    }
+    fbank.AcceptWaveform(sample_rate, buf.data(), buf.size());
+    int32_t frames = fbank.NumFramesReady();
+    for (int32_t i = 0; i != frames; ++i) {
+        const float *frame = fbank.GetFrame(i);
+        std::vector<float> frame_vector(frame, frame + fbank_opts_.mel_opts.num_bins);
+        vad_feats.emplace_back(frame_vector);
+    }
+}
+
+void FsmnVad::LoadCmvn(const char *filename)
+{
+    try{
+        using namespace std;
+        ifstream cmvn_stream(filename);
+        if (!cmvn_stream.is_open()) {
+            LOG(ERROR) << "Failed to open file: " << filename;
+            exit(-1);
+        }
+        string line;
+
+        while (getline(cmvn_stream, line)) {
+            istringstream iss(line);
+            vector<string> line_item{istream_iterator<string>{iss}, istream_iterator<string>{}};
+            if (line_item[0] == "<AddShift>") {
+                getline(cmvn_stream, line);
+                istringstream means_lines_stream(line);
+                vector<string> means_lines{istream_iterator<string>{means_lines_stream}, istream_iterator<string>{}};
+                if (means_lines[0] == "<LearnRateCoef>") {
+                    for (int j = 3; j < means_lines.size() - 1; j++) {
+                        means_list_.push_back(stof(means_lines[j]));
+                    }
+                    continue;
+                }
+            }
+            else if (line_item[0] == "<Rescale>") {
+                getline(cmvn_stream, line);
+                istringstream vars_lines_stream(line);
+                vector<string> vars_lines{istream_iterator<string>{vars_lines_stream}, istream_iterator<string>{}};
+                if (vars_lines[0] == "<LearnRateCoef>") {
+                    for (int j = 3; j < vars_lines.size() - 1; j++) {
+                        // vars_list_.push_back(stof(vars_lines[j])*scale);
+                        vars_list_.push_back(stof(vars_lines[j]));
+                    }
+                    continue;
+                }
+            }
+        }
+    }catch(std::exception const &e) {
+        LOG(ERROR) << "Error when load vad cmvn : " << e.what();
+        exit(-1);
+    }
+}
+
+void FsmnVad::LfrCmvn(std::vector<std::vector<float>> &vad_feats) {
+
+    std::vector<std::vector<float>> out_feats;
+    int T = vad_feats.size();
+    int T_lrf = ceil(1.0 * T / lfr_n);
+
+    // Pad frames at start(copy first frame)
+    for (int i = 0; i < (lfr_m - 1) / 2; i++) {
+        vad_feats.insert(vad_feats.begin(), vad_feats[0]);
+    }
+    // Merge lfr_m frames as one,lfr_n frames per window
+    T = T + (lfr_m - 1) / 2;
+    std::vector<float> p;
+    for (int i = 0; i < T_lrf; i++) {
+        if (lfr_m <= T - i * lfr_n) {
+            for (int j = 0; j < lfr_m; j++) {
+                p.insert(p.end(), vad_feats[i * lfr_n + j].begin(), vad_feats[i * lfr_n + j].end());
+            }
+            out_feats.emplace_back(p);
+            p.clear();
+        } else {
+            // Fill to lfr_m frames at last window if less than lfr_m frames  (copy last frame)
+            int num_padding = lfr_m - (T - i * lfr_n);
+            for (int j = 0; j < (vad_feats.size() - i * lfr_n); j++) {
+                p.insert(p.end(), vad_feats[i * lfr_n + j].begin(), vad_feats[i * lfr_n + j].end());
+            }
+            for (int j = 0; j < num_padding; j++) {
+                p.insert(p.end(), vad_feats[vad_feats.size() - 1].begin(), vad_feats[vad_feats.size() - 1].end());
+            }
+            out_feats.emplace_back(p);
+            p.clear();
+        }
+    }
+    // Apply cmvn
+    for (auto &out_feat: out_feats) {
+        for (int j = 0; j < means_list_.size(); j++) {
+            out_feat[j] = (out_feat[j] + means_list_[j]) * vars_list_[j];
+        }
+    }
+    vad_feats = out_feats;
+}
+
+std::vector<std::vector<int>>
+FsmnVad::Infer(std::vector<float> &waves, bool input_finished) {
+    std::vector<std::vector<float>> vad_feats;
+    std::vector<std::vector<float>> vad_probs;
+    std::vector<std::vector<int>> vad_segments;
+    FbankKaldi(vad_sample_rate_, vad_feats, waves);
+    if(vad_feats.size() == 0){
+      return vad_segments;
+    }
+    LfrCmvn(vad_feats);
+    Forward(vad_feats, &vad_probs, &in_cache_, input_finished);
+
+    E2EVadModel vad_scorer = E2EVadModel();
+    vad_segments = vad_scorer(vad_probs, waves, true, false, vad_silence_duration_, vad_max_len_,
+                              vad_speech_noise_thres_, vad_sample_rate_);
+    return vad_segments;
+}
+
+void FsmnVad::InitCache(){
+  std::vector<float> cache_feats(128 * 19 * 1, 0);
+  for (int i=0;i<4;i++){
+    in_cache_.emplace_back(cache_feats);
+  }
+};
+
+void FsmnVad::Reset(){
+  in_cache_.clear();
+  InitCache();
+};
+
+void FsmnVad::Test() {
+}
+
+FsmnVad::~FsmnVad() {
+}
+
+FsmnVad::FsmnVad():env_(ORT_LOGGING_LEVEL_ERROR, ""),session_options_{} {
+}
+
+} // namespace funasr

runtime/onnxruntime/src/fsmn-vad.h

@@ -0,0 +1,68 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#ifndef VAD_SERVER_FSMNVAD_H
+#define VAD_SERVER_FSMNVAD_H
+
+#include "precomp.h"
+
+namespace funasr {
+class FsmnVad : public VadModel {
+/**
+ * Author: Speech Lab of DAMO Academy, Alibaba Group
+ * Deep-FSMN for Large Vocabulary Continuous Speech Recognition
+ * https://arxiv.org/abs/1803.05030
+*/
+
+public:
+    FsmnVad();
+    ~FsmnVad();
+    void Test();
+    void InitVad(const std::string &vad_model, const std::string &vad_cmvn, const std::string &vad_config, int thread_num);
+    std::vector<std::vector<int>> Infer(std::vector<float> &waves, bool input_finished=true);
+    void Forward(
+        const std::vector<std::vector<float>> &chunk_feats,
+        std::vector<std::vector<float>> *out_prob,
+        std::vector<std::vector<float>> *in_cache,
+        bool is_final);
+    void Reset();
+
+    int GetVadSampleRate() { return vad_sample_rate_; };
+    
+    std::shared_ptr<Ort::Session> vad_session_ = nullptr;
+    Ort::Env env_;
+    Ort::SessionOptions session_options_;
+    vector<string> m_strInputNames, m_strOutputNames;
+    std::vector<const char *> vad_in_names_;
+    std::vector<const char *> vad_out_names_;
+    std::vector<std::vector<float>> in_cache_;
+    
+    knf::FbankOptions fbank_opts_;
+    std::vector<float> means_list_;
+    std::vector<float> vars_list_;
+
+    int vad_sample_rate_ = MODEL_SAMPLE_RATE;
+    int vad_silence_duration_ = VAD_SILENCE_DURATION;
+    int vad_max_len_ = VAD_MAX_LEN;
+    double vad_speech_noise_thres_ = VAD_SPEECH_NOISE_THRES;
+    int lfr_m = VAD_LFR_M;
+    int lfr_n = VAD_LFR_N;
+
+private:
+
+    void ReadModel(const char* vad_model);
+    void LoadConfigFromYaml(const char* filename);
+
+    void FbankKaldi(float sample_rate, std::vector<std::vector<float>> &vad_feats,
+                    std::vector<float> &waves);
+
+    void LfrCmvn(std::vector<std::vector<float>> &vad_feats);
+    void LoadCmvn(const char *filename);
+    void InitCache();
+
+};
+
+} // namespace funasr
+#endif //VAD_SERVER_FSMNVAD_H

runtime/onnxruntime/src/funasrruntime.cpp

@@ -0,0 +1,913 @@
+#include "precomp.h"
+#include <vector>
+
+
+	// APIs for Init
+	_FUNASRAPI FUNASR_HANDLE  FunASRInit(std::map<std::string, std::string>& model_path, int thread_num, ASR_TYPE type)
+	{
+		funasr::Model* mm = funasr::CreateModel(model_path, thread_num, type);
+		return mm;
+	}
+
+	_FUNASRAPI FUNASR_HANDLE  FunASROnlineInit(FUNASR_HANDLE asr_hanlde, std::vector<int> chunk_size)
+	{
+		funasr::Model* mm = funasr::CreateModel(asr_hanlde, chunk_size);
+		return mm;
+	}
+
+	_FUNASRAPI FUNASR_HANDLE  FsmnVadInit(std::map<std::string, std::string>& model_path, int thread_num)
+	{
+		funasr::VadModel* mm = funasr::CreateVadModel(model_path, thread_num);
+		return mm;
+	}
+
+	_FUNASRAPI FUNASR_HANDLE  FsmnVadOnlineInit(FUNASR_HANDLE fsmnvad_handle)
+	{
+		funasr::VadModel* mm = funasr::CreateVadModel(fsmnvad_handle);
+		return mm;
+	}
+
+	_FUNASRAPI FUNASR_HANDLE  CTTransformerInit(std::map<std::string, std::string>& model_path, int thread_num, PUNC_TYPE type)
+	{
+		funasr::PuncModel* mm = funasr::CreatePuncModel(model_path, thread_num, type);
+		return mm;
+	}
+
+	_FUNASRAPI FUNASR_HANDLE  FunOfflineInit(std::map<std::string, std::string>& model_path, int thread_num, bool use_gpu, int batch_size)
+	{
+		funasr::OfflineStream* mm = funasr::CreateOfflineStream(model_path, thread_num, use_gpu, batch_size);
+		return mm;
+	}
+
+	_FUNASRAPI FUNASR_HANDLE  FunTpassInit(std::map<std::string, std::string>& model_path, int thread_num)
+	{
+		funasr::TpassStream* mm = funasr::CreateTpassStream(model_path, thread_num);
+		return mm;
+	}
+
+	_FUNASRAPI FUNASR_HANDLE FunTpassOnlineInit(FUNASR_HANDLE tpass_handle, std::vector<int> chunk_size)
+	{
+		return funasr::CreateTpassOnlineStream(tpass_handle, chunk_size);
+	}
+
+	// APIs for ASR Infer
+	_FUNASRAPI FUNASR_RESULT FunASRInferBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len, FUNASR_MODE mode, QM_CALLBACK fn_callback, bool input_finished, int sampling_rate, std::string wav_format)
+	{
+		funasr::Model* recog_obj = (funasr::Model*)handle;
+		if (!recog_obj)
+			return nullptr;
+
+		funasr::Audio audio(recog_obj->GetAsrSampleRate(),1);
+		if(wav_format == "pcm" || wav_format == "PCM"){
+			if (!audio.LoadPcmwav(sz_buf, n_len, &sampling_rate))
+				return nullptr;
+		}else{
+			if (!audio.FfmpegLoad(sz_buf, n_len))
+				return nullptr;
+		}
+
+		float* buff;
+		int len;
+		int flag = 0;
+		funasr::FUNASR_RECOG_RESULT* p_result = new funasr::FUNASR_RECOG_RESULT;
+		p_result->snippet_time = audio.GetTimeLen();
+		if(p_result->snippet_time == 0){
+			return p_result;
+		}
+
+		while (audio.Fetch(buff, len, flag) > 0) {
+			string msg = recog_obj->Forward(buff, len, input_finished);
+			p_result->msg += msg;
+		}
+		return p_result;
+	}
+
+	_FUNASRAPI FUNASR_RESULT FunASRInfer(FUNASR_HANDLE handle, const char* sz_filename, FUNASR_MODE mode, QM_CALLBACK fn_callback, int sampling_rate)
+	{
+		funasr::Model* recog_obj = (funasr::Model*)handle;
+		if (!recog_obj)
+			return nullptr;
+
+		funasr::Audio audio(recog_obj->GetAsrSampleRate(),1);
+		if(funasr::is_target_file(sz_filename, "wav")){
+			int32_t sampling_rate_ = -1;
+			if(!audio.LoadWav(sz_filename, &sampling_rate_))
+				return nullptr;
+		}else if(funasr::is_target_file(sz_filename, "pcm")){
+			if (!audio.LoadPcmwav(sz_filename, &sampling_rate))
+				return nullptr;
+		}else{
+			if (!audio.FfmpegLoad(sz_filename))
+				return nullptr;
+		}
+
+		float* buff;
+		int len;
+		int flag = 0;
+		funasr::FUNASR_RECOG_RESULT* p_result = new funasr::FUNASR_RECOG_RESULT;
+		p_result->snippet_time = audio.GetTimeLen();
+		if(p_result->snippet_time == 0){
+            return p_result;
+        }
+		while (audio.Fetch(buff, len, flag) > 0) {
+			string msg = recog_obj->Forward(buff, len, true);
+			p_result->msg += msg;
+		}
+		return p_result;
+	}
+
+	// APIs for VAD Infer
+	_FUNASRAPI FUNASR_RESULT FsmnVadInferBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len, QM_CALLBACK fn_callback, bool input_finished, int sampling_rate, std::string wav_format)
+	{
+		funasr::VadModel* vad_obj = (funasr::VadModel*)handle;
+		if (!vad_obj)
+			return nullptr;
+
+		funasr::Audio audio(vad_obj->GetVadSampleRate(),1);
+		if(wav_format == "pcm" || wav_format == "PCM"){
+			if (!audio.LoadPcmwav(sz_buf, n_len, &sampling_rate))
+				return nullptr;
+		}else{
+			if (!audio.FfmpegLoad(sz_buf, n_len))
+				return nullptr;
+		}
+
+		funasr::FUNASR_VAD_RESULT* p_result = new funasr::FUNASR_VAD_RESULT;
+		p_result->snippet_time = audio.GetTimeLen();
+		if(p_result->snippet_time == 0){
+			p_result->segments = new vector<std::vector<int>>();
+            return p_result;
+        }
+		
+		vector<std::vector<int>> vad_segments;
+		audio.Split(vad_obj, vad_segments, input_finished);
+		p_result->segments = new vector<std::vector<int>>(vad_segments);
+
+		return p_result;
+	}
+
+	_FUNASRAPI FUNASR_RESULT FsmnVadInfer(FUNASR_HANDLE handle, const char* sz_filename, QM_CALLBACK fn_callback, int sampling_rate)
+	{
+		funasr::VadModel* vad_obj = (funasr::VadModel*)handle;
+		if (!vad_obj)
+			return nullptr;
+
+		funasr::Audio audio(vad_obj->GetVadSampleRate(),1);
+		if(funasr::is_target_file(sz_filename, "wav")){
+			int32_t sampling_rate_ = -1;
+			if(!audio.LoadWav(sz_filename, &sampling_rate_))
+				return nullptr;
+		}else if(funasr::is_target_file(sz_filename, "pcm")){
+			if (!audio.LoadPcmwav(sz_filename, &sampling_rate))
+				return nullptr;
+		}else{
+			if (!audio.FfmpegLoad(sz_filename))
+				return nullptr;
+		}
+
+		funasr::FUNASR_VAD_RESULT* p_result = new funasr::FUNASR_VAD_RESULT;
+		p_result->snippet_time = audio.GetTimeLen();
+		if(p_result->snippet_time == 0){
+			p_result->segments = new vector<std::vector<int>>();
+            return p_result;
+        }
+		
+		vector<std::vector<int>> vad_segments;
+		audio.Split(vad_obj, vad_segments, true);
+		p_result->segments = new vector<std::vector<int>>(vad_segments);
+
+		return p_result;
+	}
+
+	// APIs for PUNC Infer
+	_FUNASRAPI FUNASR_RESULT CTTransformerInfer(FUNASR_HANDLE handle, const char* sz_sentence, FUNASR_MODE mode, QM_CALLBACK fn_callback, PUNC_TYPE type, FUNASR_RESULT pre_result)
+	{
+		funasr::PuncModel* punc_obj = (funasr::PuncModel*)handle;
+		if (!punc_obj)
+			return nullptr;
+		
+		FUNASR_RESULT p_result = nullptr;
+		if (type==PUNC_OFFLINE){
+			p_result = (FUNASR_RESULT)new funasr::FUNASR_PUNC_RESULT;
+			((funasr::FUNASR_PUNC_RESULT*)p_result)->msg = punc_obj->AddPunc(sz_sentence);
+		}else if(type==PUNC_ONLINE){
+			if (!pre_result)
+				p_result = (FUNASR_RESULT)new funasr::FUNASR_PUNC_RESULT;
+			else
+				p_result = pre_result;
+			((funasr::FUNASR_PUNC_RESULT*)p_result)->msg = punc_obj->AddPunc(sz_sentence, ((funasr::FUNASR_PUNC_RESULT*)p_result)->arr_cache);
+		}else{
+			LOG(ERROR) << "Wrong PUNC_TYPE";
+			exit(-1);
+		}
+
+		return p_result;
+	}
+
+	// APIs for Offline-stream Infer
+	_FUNASRAPI FUNASR_RESULT FunOfflineInferBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len, 
+												   FUNASR_MODE mode, QM_CALLBACK fn_callback, const std::vector<std::vector<float>> &hw_emb, 
+												   int sampling_rate, std::string wav_format, bool itn, FUNASR_DEC_HANDLE dec_handle,
+												   std::string svs_lang, bool svs_itn)
+	{
+		funasr::OfflineStream* offline_stream = (funasr::OfflineStream*)handle;
+		if (!offline_stream)
+			return nullptr;
+
+		funasr::Audio audio(offline_stream->asr_handle->GetAsrSampleRate(),1);
+		try{
+			if(wav_format == "pcm" || wav_format == "PCM"){
+				if (!audio.LoadPcmwav(sz_buf, n_len, &sampling_rate))
+					return nullptr;
+			}else{
+				if (!audio.FfmpegLoad(sz_buf, n_len))
+					return nullptr;
+			}
+		}catch (std::exception const &e)
+		{
+			LOG(ERROR)<<e.what();
+			return nullptr;
+		}
+
+		funasr::FUNASR_RECOG_RESULT* p_result = new funasr::FUNASR_RECOG_RESULT;
+		p_result->snippet_time = audio.GetTimeLen();
+		if(p_result->snippet_time == 0){
+            return p_result;
+        }
+		std::vector<int> index_vector={0};
+		int msg_idx = 0;
+		if(offline_stream->UseVad()){
+			audio.CutSplit(offline_stream, index_vector);
+		}
+		std::vector<string> msgs(index_vector.size());
+		std::vector<float> msg_stimes(index_vector.size());
+
+		float** buff;
+		int* len;
+		int* flag;
+		float* start_time;
+		int batch_size = offline_stream->asr_handle->GetBatchSize();
+		int batch_in = 0;
+
+		std::string cur_stamp = "[";
+		std::string lang = (offline_stream->asr_handle)->GetLang();
+		while (audio.FetchDynamic(buff, len, flag, start_time, batch_size, batch_in) > 0) {
+			// dec reset
+			funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)dec_handle;
+			if (wfst_decoder){
+				wfst_decoder->StartUtterance();
+			}
+			vector<string> msg_batch;
+			if(offline_stream->GetModelType() == MODEL_SVS){
+				msg_batch = (offline_stream->asr_handle)->Forward(buff, len, true, svs_lang, svs_itn, batch_in);
+			}else{
+				msg_batch = (offline_stream->asr_handle)->Forward(buff, len, true, hw_emb, dec_handle, batch_in);
+			}
+			for(int idx=0; idx<batch_in; idx++){
+				string msg = msg_batch[idx];
+				if(msg_idx < index_vector.size()){
+					msgs[index_vector[msg_idx]] = msg;
+					msg_stimes[index_vector[msg_idx]] = start_time[idx];
+					msg_idx++;
+				}else{
+					LOG(ERROR) << "msg_idx: " << msg_idx <<" is out of range " << index_vector.size();
+				}				
+			}
+
+			// release
+			delete[] buff;
+			buff = nullptr;
+			delete[] len;
+			len = nullptr;
+			delete[] flag;
+			flag = nullptr;
+			delete[] start_time;
+			start_time = nullptr;
+		}
+		for(int idx=0; idx<msgs.size(); idx++){
+			string msg = msgs[idx];
+			std::vector<std::string> msg_vec = funasr::SplitStr(msg, " | ");
+			if(msg_vec.size()==0){
+				continue;
+			}
+			if(lang == "en-bpe" && p_result->msg != ""){
+				p_result->msg += " ";
+			}
+			p_result->msg += msg_vec[0];
+			//timestamp
+			if(msg_vec.size() > 1){
+				std::vector<std::string> msg_stamp = funasr::split(msg_vec[1], ',');
+				for(size_t i = 0; i + 1 < msg_stamp.size(); i += 2){
+					float begin = std::stof(msg_stamp[i])+msg_stimes[idx];
+					float end = std::stof(msg_stamp[i+1])+msg_stimes[idx];
+					cur_stamp += "["+std::to_string((int)(1000*begin))+","+std::to_string((int)(1000*end))+"],";
+				}
+			}
+		}
+		if(cur_stamp != "["){
+			cur_stamp.erase(cur_stamp.length() - 1);
+			p_result->stamp += cur_stamp + "]";
+		}
+		if(offline_stream->UsePunc()){
+			string punc_res = (offline_stream->punc_handle)->AddPunc((p_result->msg).c_str(), lang);
+			p_result->msg = punc_res;
+		}
+#if !defined(__APPLE__)
+		if(offline_stream->UseITN() && itn){
+			string msg_itn = offline_stream->itn_handle->Normalize(p_result->msg);
+			if(!(p_result->stamp).empty()){
+				std::string new_stamp = funasr::TimestampSmooth(p_result->msg, msg_itn, p_result->stamp);
+				if(!new_stamp.empty()){
+					p_result->stamp = new_stamp;
+				}
+			}			
+			p_result->msg = msg_itn;
+		}
+#endif
+		if (!(p_result->stamp).empty()){
+			p_result->stamp_sents = funasr::TimestampSentence(p_result->msg, p_result->stamp);
+		}
+		return p_result;
+	}
+
+	_FUNASRAPI FUNASR_RESULT FunOfflineInfer(FUNASR_HANDLE handle, const char* sz_filename, FUNASR_MODE mode, QM_CALLBACK fn_callback, 
+											 const std::vector<std::vector<float>> &hw_emb, int sampling_rate, bool itn, FUNASR_DEC_HANDLE dec_handle)
+	{
+		funasr::OfflineStream* offline_stream = (funasr::OfflineStream*)handle;
+		if (!offline_stream)
+			return nullptr;
+		
+		funasr::Audio audio((offline_stream->asr_handle)->GetAsrSampleRate(),1);
+		try{
+			if(funasr::is_target_file(sz_filename, "wav")){
+				int32_t sampling_rate_ = -1;
+				if(!audio.LoadWav(sz_filename, &sampling_rate_))
+					return nullptr;
+			}else if(funasr::is_target_file(sz_filename, "pcm")){
+				if (!audio.LoadPcmwav(sz_filename, &sampling_rate))
+					return nullptr;
+			}else{
+				if (!audio.FfmpegLoad(sz_filename))
+					return nullptr;
+			}
+		}catch (std::exception const &e)
+		{
+			LOG(ERROR)<<e.what();
+			return nullptr;
+		}
+		
+		funasr::FUNASR_RECOG_RESULT* p_result = new funasr::FUNASR_RECOG_RESULT;
+		p_result->snippet_time = audio.GetTimeLen();
+		if(p_result->snippet_time == 0){
+            return p_result;
+        }
+		std::vector<int> index_vector={0};
+		int msg_idx = 0;
+		if(offline_stream->UseVad()){
+			audio.CutSplit(offline_stream, index_vector);
+		}
+		std::vector<string> msgs(index_vector.size());
+		std::vector<float> msg_stimes(index_vector.size());
+
+		float** buff;
+		int* len;
+		int* flag;
+		float* start_time;
+		int batch_size = offline_stream->asr_handle->GetBatchSize();
+		int batch_in = 0;
+
+		std::string cur_stamp = "[";
+		std::string lang = (offline_stream->asr_handle)->GetLang();
+		while (audio.FetchDynamic(buff, len, flag, start_time, batch_size, batch_in) > 0) {
+			// dec reset
+			funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)dec_handle;
+			if (wfst_decoder){
+				wfst_decoder->StartUtterance();
+			}
+			vector<string> msg_batch = (offline_stream->asr_handle)->Forward(buff, len, true, hw_emb, dec_handle, batch_in);
+			for(int idx=0; idx<batch_in; idx++){
+				string msg = msg_batch[idx];
+				if(msg_idx < index_vector.size()){
+					msgs[index_vector[msg_idx]] = msg;
+					msg_stimes[index_vector[msg_idx]] = start_time[idx];
+					msg_idx++;
+				}else{
+					LOG(ERROR) << "msg_idx: " << msg_idx <<" is out of range " << index_vector.size();
+				}				
+			}
+
+			// release
+			delete[] buff;
+			buff = nullptr;
+			delete[] len;
+			len = nullptr;
+			delete[] flag;
+			flag = nullptr;
+			delete[] start_time;
+			start_time = nullptr;
+		}
+		for(int idx=0; idx<msgs.size(); idx++){
+			string msg = msgs[idx];
+			std::vector<std::string> msg_vec = funasr::SplitStr(msg, " | ");
+			if(msg_vec.size()==0){
+				continue;
+			}
+			if(lang == "en-bpe" && p_result->msg != ""){
+				p_result->msg += " ";
+			}
+			p_result->msg += msg_vec[0];
+			//timestamp
+			if(msg_vec.size() > 1){
+				std::vector<std::string> msg_stamp = funasr::split(msg_vec[1], ',');
+				for(size_t i = 0; i + 1 < msg_stamp.size(); i += 2){
+					float begin = std::stof(msg_stamp[i])+msg_stimes[idx];
+					float end = std::stof(msg_stamp[i+1])+msg_stimes[idx];
+					cur_stamp += "["+std::to_string((int)(1000*begin))+","+std::to_string((int)(1000*end))+"],";
+				}
+			}
+		}
+		if(cur_stamp != "["){
+			cur_stamp.erase(cur_stamp.length() - 1);
+			p_result->stamp += cur_stamp + "]";
+		}
+		if(offline_stream->UsePunc()){
+			string punc_res = (offline_stream->punc_handle)->AddPunc((p_result->msg).c_str(), lang);
+			p_result->msg = punc_res;
+		}
+#if !defined(__APPLE__)
+		if(offline_stream->UseITN() && itn){
+			string msg_itn = offline_stream->itn_handle->Normalize(p_result->msg);
+			if(!(p_result->stamp).empty()){
+				std::string new_stamp = funasr::TimestampSmooth(p_result->msg, msg_itn, p_result->stamp);
+				if(!new_stamp.empty()){
+					p_result->stamp = new_stamp;
+				}
+			}
+			p_result->msg = msg_itn;
+		}
+#endif
+		if (!(p_result->stamp).empty()){
+			p_result->stamp_sents = funasr::TimestampSentence(p_result->msg, p_result->stamp);
+		}
+		return p_result;
+	}
+
+//#if !defined(__APPLE__)
+	_FUNASRAPI const std::vector<std::vector<float>> CompileHotwordEmbedding(FUNASR_HANDLE handle, std::string &hotwords, ASR_TYPE mode)
+	{
+		if (mode == ASR_OFFLINE){
+			funasr::OfflineStream* offline_stream = (funasr::OfflineStream*)handle;
+    		std::vector<std::vector<float>> emb;
+			if (!offline_stream)
+				return emb;
+			return (offline_stream->asr_handle)->CompileHotwordEmbedding(hotwords);
+		}
+		else if (mode == ASR_TWO_PASS){
+			funasr::TpassStream* tpass_stream = (funasr::TpassStream*)handle;
+    		std::vector<std::vector<float>> emb;
+			if (!tpass_stream)
+				return emb;
+			return (tpass_stream->asr_handle)->CompileHotwordEmbedding(hotwords);
+		}
+		else{
+			LOG(ERROR) << "Not implement: Online model does not support Hotword yet!";
+			std::vector<std::vector<float>> emb;
+			return emb;
+		}
+		
+	}
+//#endif
+
+	// APIs for 2pass-stream Infer
+	_FUNASRAPI FUNASR_RESULT FunTpassInferBuffer(FUNASR_HANDLE handle, FUNASR_HANDLE online_handle, const char* sz_buf, 
+												 int n_len, std::vector<std::vector<std::string>> &punc_cache, bool input_finished, 
+												 int sampling_rate, std::string wav_format, ASR_TYPE mode, 
+												 const std::vector<std::vector<float>> &hw_emb, bool itn, FUNASR_DEC_HANDLE dec_handle,
+												 std::string svs_lang, bool svs_itn)
+	{
+		funasr::TpassStream* tpass_stream = (funasr::TpassStream*)handle;
+		funasr::TpassOnlineStream* tpass_online_stream = (funasr::TpassOnlineStream*)online_handle;
+		if (!tpass_stream || !tpass_online_stream)
+			return nullptr;
+		
+		funasr::VadModel* vad_online_handle = (tpass_online_stream->vad_online_handle).get();
+		if (!vad_online_handle)
+			return nullptr;
+
+		funasr::Audio* audio = ((funasr::FsmnVadOnline*)vad_online_handle)->audio_handle.get();
+
+		funasr::Model* asr_online_handle = (tpass_online_stream->asr_online_handle).get();
+		if (!asr_online_handle)
+			return nullptr;
+		int chunk_len = ((funasr::ParaformerOnline*)asr_online_handle)->chunk_len;
+		
+		funasr::Model* asr_handle = (tpass_stream->asr_handle).get();
+		if (!asr_handle)
+			return nullptr;
+
+		funasr::PuncModel* punc_online_handle = (tpass_stream->punc_online_handle).get();
+		if (!punc_online_handle)
+			return nullptr;
+
+		if(wav_format == "pcm" || wav_format == "PCM"){
+			if (!audio->LoadPcmwavOnline(sz_buf, n_len, &sampling_rate))
+				return nullptr;
+		}else{
+			// if (!audio->FfmpegLoad(sz_buf, n_len))
+			// 	return nullptr;
+			LOG(ERROR) <<"Wrong wav_format: " << wav_format ;
+			return nullptr;
+		}
+
+		funasr::FUNASR_RECOG_RESULT* p_result = new funasr::FUNASR_RECOG_RESULT;
+		p_result->snippet_time = audio->GetTimeLen();
+		
+		audio->Split(vad_online_handle, chunk_len, input_finished, mode);
+		p_result->start = audio->start;
+		p_result->end = audio->end;
+
+		funasr::AudioFrame* frame = nullptr;
+		while(audio->FetchChunck(frame) > 0){
+			string msg = (asr_online_handle)->Forward(frame->data, frame->len, frame->is_final);
+			if(mode == ASR_ONLINE){
+				((funasr::ParaformerOnline*)asr_online_handle)->online_res += msg;
+				if(frame->is_final){
+					string online_msg = ((funasr::ParaformerOnline*)asr_online_handle)->online_res;
+					string msg_punc = punc_online_handle->AddPunc(online_msg.c_str(), punc_cache[0]);
+					p_result->tpass_msg = msg_punc;
+#if !defined(__APPLE__)
+					// ITN
+					if(tpass_stream->UseITN() && itn){
+						string msg_itn = tpass_stream->itn_handle->Normalize(msg_punc);
+						p_result->tpass_msg = msg_itn;
+					}
+#endif
+					((funasr::ParaformerOnline*)asr_online_handle)->online_res = "";
+					p_result->msg += msg;
+				}else{
+					p_result->msg += msg;
+				}
+			}else if(mode == ASR_TWO_PASS){
+				p_result->msg += msg;
+			}
+			if(frame != nullptr){
+				delete frame;
+				frame = nullptr;
+			}
+		}
+
+		// timestamp
+		std::string cur_stamp = "[";		
+		while(audio->FetchTpass(frame) > 0){
+			// dec reset
+			funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)dec_handle;
+			if (wfst_decoder){
+				wfst_decoder->StartUtterance();
+			}
+			float** buff;
+			int* len;
+			buff = new float*[1];
+        	len = new int[1];
+			buff[0] = frame->data;
+			len[0] = frame->len;
+			vector<string> msgs;
+			if(tpass_stream->GetModelType() == MODEL_SVS){
+				msgs = (tpass_stream->asr_handle)->Forward(buff, len, true, svs_lang, svs_itn, 1);
+			}else{
+				msgs = (tpass_stream->asr_handle)->Forward(buff, len, true, hw_emb, dec_handle, 1);
+			}
+			string msg = msgs.size()>0?msgs[0]:"";
+			std::vector<std::string> msg_vec = funasr::SplitStr(msg, " | ");  // split with timestamp
+			if(msg_vec.size()==0){
+				continue;
+			}
+			msg = msg_vec[0];
+			//timestamp
+			if(msg_vec.size() > 1){
+				std::vector<std::string> msg_stamp = funasr::split(msg_vec[1], ',');
+				for(size_t i = 0; i + 1 < msg_stamp.size(); i += 2){
+					float begin = std::stof(msg_stamp[i]) + float(frame->global_start)/1000.0;
+					float end = std::stof(msg_stamp[i+1]) + float(frame->global_start)/1000.0;
+					cur_stamp += "["+std::to_string((int)(1000*begin))+","+std::to_string((int)(1000*end))+"],";
+				}
+			}
+
+			if(cur_stamp != "["){
+				cur_stamp.erase(cur_stamp.length() - 1);
+				p_result->stamp += cur_stamp + "]";
+			}
+
+			if (tpass_stream->GetModelType() == MODEL_PARA){
+				string msg_punc = punc_online_handle->AddPunc(msg.c_str(), punc_cache[1]);
+				if(input_finished){
+					msg_punc += "。";
+				}
+				p_result->tpass_msg = msg_punc;
+
+#if !defined(__APPLE__)
+				if(tpass_stream->UseITN() && itn){
+					string msg_itn = tpass_stream->itn_handle->Normalize(msg_punc);
+					// TimestampSmooth
+					if(!(p_result->stamp).empty()){
+						std::string new_stamp = funasr::TimestampSmooth(p_result->tpass_msg, msg_itn, p_result->stamp);
+						if(!new_stamp.empty()){
+							p_result->stamp = new_stamp;
+						}
+					}
+					p_result->tpass_msg = msg_itn;
+				}
+#endif
+			}else{
+				p_result->tpass_msg = msg;
+			}
+			if (!(p_result->stamp).empty()){
+				p_result->stamp_sents = funasr::TimestampSentence(p_result->tpass_msg, p_result->stamp);
+			}
+			if(frame != nullptr){
+				delete frame;
+				frame = nullptr;
+			}
+		}
+
+		if(input_finished){
+			audio->ResetIndex();
+		}
+
+		return p_result;
+	}
+
+	_FUNASRAPI const int FunASRGetRetNumber(FUNASR_RESULT result)
+	{
+		if (!result)
+			return 0;
+		return 1;
+	}
+
+	// APIs for GetRetSnippetTime
+	_FUNASRAPI const float FunASRGetRetSnippetTime(FUNASR_RESULT result)
+	{
+		if (!result)
+			return 0.0f;
+
+		return ((funasr::FUNASR_RECOG_RESULT*)result)->snippet_time;
+	}
+
+	_FUNASRAPI const float FsmnVadGetRetSnippetTime(FUNASR_RESULT result)
+	{
+		if (!result)
+			return 0.0f;
+
+		return ((funasr::FUNASR_VAD_RESULT*)result)->snippet_time;
+	}
+
+	// APIs for GetResult
+	_FUNASRAPI const char* FunASRGetResult(FUNASR_RESULT result,int n_index)
+	{
+		funasr::FUNASR_RECOG_RESULT * p_result = (funasr::FUNASR_RECOG_RESULT*)result;
+		if(!p_result)
+			return nullptr;
+
+		return p_result->msg.c_str();
+	}
+
+	_FUNASRAPI const char* FunASRGetStamp(FUNASR_RESULT result)
+	{
+		funasr::FUNASR_RECOG_RESULT * p_result = (funasr::FUNASR_RECOG_RESULT*)result;
+		if(!p_result)
+			return nullptr;
+
+		return p_result->stamp.c_str();
+	}
+
+		_FUNASRAPI const char* FunASRGetStampSents(FUNASR_RESULT result)
+	{
+		funasr::FUNASR_RECOG_RESULT * p_result = (funasr::FUNASR_RECOG_RESULT*)result;
+		if(!p_result)
+			return nullptr;
+
+		return p_result->stamp_sents.c_str();
+	}
+
+	_FUNASRAPI const char* FunASRGetTpassResult(FUNASR_RESULT result,int n_index)
+	{
+		funasr::FUNASR_RECOG_RESULT * p_result = (funasr::FUNASR_RECOG_RESULT*)result;
+		if(!p_result)
+			return nullptr;
+
+		return p_result->tpass_msg.c_str();
+	}
+
+	_FUNASRAPI const int64_t FunASRGetTpassStart(FUNASR_RESULT result)
+	{
+		funasr::FUNASR_RECOG_RESULT * p_result = (funasr::FUNASR_RECOG_RESULT*)result;
+		if(!p_result)
+			return 0;
+
+		return p_result->start;
+	}
+	_FUNASRAPI const int64_t FunASRGetTpassEnd(FUNASR_RESULT result)
+	{
+		funasr::FUNASR_RECOG_RESULT * p_result = (funasr::FUNASR_RECOG_RESULT*)result;
+		if(!p_result)
+			return 0;
+
+		return p_result->end;
+	}
+
+	_FUNASRAPI const char* CTTransformerGetResult(FUNASR_RESULT result,int n_index)
+	{
+		funasr::FUNASR_PUNC_RESULT * p_result = (funasr::FUNASR_PUNC_RESULT*)result;
+		if(!p_result)
+			return nullptr;
+
+		return p_result->msg.c_str();
+	}
+
+	_FUNASRAPI vector<std::vector<int>>* FsmnVadGetResult(FUNASR_RESULT result,int n_index)
+	{
+		funasr::FUNASR_VAD_RESULT * p_result = (funasr::FUNASR_VAD_RESULT*)result;
+		if(!p_result)
+			return nullptr;
+
+		return p_result->segments;
+	}
+
+	// APIs for FreeResult
+	_FUNASRAPI void FunASRFreeResult(FUNASR_RESULT result)
+	{
+		if (result)
+		{
+			delete (funasr::FUNASR_RECOG_RESULT*)result;
+		}
+	}
+
+	_FUNASRAPI void CTTransformerFreeResult(FUNASR_RESULT result)
+	{
+		if (result)
+		{
+			delete (funasr::FUNASR_PUNC_RESULT*)result;
+		}
+	}
+
+	_FUNASRAPI void FsmnVadFreeResult(FUNASR_RESULT result)
+	{
+		funasr::FUNASR_VAD_RESULT * p_result = (funasr::FUNASR_VAD_RESULT*)result;
+		if (p_result)
+		{
+			if(p_result->segments){
+				delete p_result->segments;
+			}
+			delete p_result;
+		}
+	}
+
+        // APIs for decoder status reset
+	_FUNASRAPI void FunASRReset(FUNASR_HANDLE handle, FUNASR_DEC_HANDLE dec_handle)
+	{
+		funasr::Model* recog_obj = (funasr::Model*)handle;
+		recog_obj->StartUtterance();
+		funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)dec_handle;
+		if (wfst_decoder)
+			wfst_decoder->StartUtterance();
+	}
+
+	_FUNASRAPI void FunOfflineReset(FUNASR_HANDLE handle, FUNASR_DEC_HANDLE dec_handle)
+	{
+		funasr::OfflineStream* recog_obj = (funasr::OfflineStream*)handle;
+		recog_obj->asr_handle->StartUtterance();
+		funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)dec_handle;
+		if (wfst_decoder)
+			wfst_decoder->StartUtterance();
+	}
+
+	// APIs for Uninit
+	_FUNASRAPI void FunASRUninit(FUNASR_HANDLE handle)
+	{
+		funasr::Model* recog_obj = (funasr::Model*)handle;
+
+		if (!recog_obj)
+			return;
+
+		delete recog_obj;
+	}
+
+	_FUNASRAPI void FsmnVadUninit(FUNASR_HANDLE handle)
+	{
+		funasr::VadModel* recog_obj = (funasr::VadModel*)handle;
+
+		if (!recog_obj)
+			return;
+
+		delete recog_obj;
+	}
+
+	_FUNASRAPI void CTTransformerUninit(FUNASR_HANDLE handle)
+	{
+		funasr::PuncModel* punc_obj = (funasr::PuncModel*)handle;
+
+		if (!punc_obj)
+			return;
+
+		delete punc_obj;
+	}
+
+	_FUNASRAPI void FunOfflineUninit(FUNASR_HANDLE handle)
+	{
+		funasr::OfflineStream* offline_stream = (funasr::OfflineStream*)handle;
+
+		if (!offline_stream)
+			return;
+
+		delete offline_stream;
+	}
+
+	_FUNASRAPI void FunTpassUninit(FUNASR_HANDLE handle)
+	{
+		funasr::TpassStream* tpass_stream = (funasr::TpassStream*)handle;
+
+		if (!tpass_stream)
+			return;
+
+		delete tpass_stream;
+	}
+
+	_FUNASRAPI void FunTpassOnlineUninit(FUNASR_HANDLE handle)
+	{
+		funasr::TpassOnlineStream* tpass_online_stream = (funasr::TpassOnlineStream*)handle;
+
+		if (!tpass_online_stream)
+			return;
+
+		delete tpass_online_stream;
+	}
+
+	_FUNASRAPI FUNASR_DEC_HANDLE FunASRWfstDecoderInit(FUNASR_HANDLE handle, int asr_type, float glob_beam, float lat_beam, float am_scale)
+	{
+		funasr::WfstDecoder* mm = nullptr;
+		if (asr_type == ASR_OFFLINE) {
+			funasr::OfflineStream* offline_stream = (funasr::OfflineStream*)handle;
+			auto paraformer = dynamic_cast<funasr::Paraformer*>(offline_stream->asr_handle.get());
+			if(paraformer !=nullptr){
+				if (paraformer->lm_){
+					mm = new funasr::WfstDecoder(paraformer->lm_.get(),
+						paraformer->GetPhoneSet(), paraformer->GetLmVocab(), glob_beam, lat_beam, am_scale);
+				}
+				return mm;
+			}
+			#ifdef USE_GPU
+			auto paraformer_torch = dynamic_cast<funasr::ParaformerTorch*>(offline_stream->asr_handle.get());
+			if(paraformer_torch !=nullptr){
+				if (paraformer_torch->lm_){
+					mm = new funasr::WfstDecoder(paraformer_torch->lm_.get(),
+						paraformer_torch->GetPhoneSet(), paraformer_torch->GetLmVocab(), glob_beam, lat_beam, am_scale);
+				}
+				return mm;
+			}
+			#endif
+
+		} else if (asr_type == ASR_TWO_PASS){
+			funasr::TpassStream* tpass_stream = (funasr::TpassStream*)handle;
+			auto paraformer = dynamic_cast<funasr::Paraformer*>(tpass_stream->asr_handle.get());
+			if(paraformer !=nullptr){
+				if (paraformer->lm_){
+					mm = new funasr::WfstDecoder(paraformer->lm_.get(),
+						paraformer->GetPhoneSet(), paraformer->GetLmVocab(), glob_beam, lat_beam, am_scale);
+				}
+				return mm;
+			}
+			#ifdef USE_GPU
+			auto paraformer_torch = dynamic_cast<funasr::ParaformerTorch*>(tpass_stream->asr_handle.get());
+			if(paraformer_torch !=nullptr){
+				if (paraformer_torch->lm_){
+					mm = new funasr::WfstDecoder(paraformer_torch->lm_.get(),
+						paraformer_torch->GetPhoneSet(), paraformer_torch->GetLmVocab(), glob_beam, lat_beam, am_scale);
+				}
+				return mm;
+			}
+			#endif
+		}
+		return mm;
+	}
+
+	_FUNASRAPI void FunASRWfstDecoderUninit(FUNASR_DEC_HANDLE handle)
+	{
+		funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)handle;
+		if (!wfst_decoder)
+			return;
+		delete wfst_decoder;
+	}
+
+	_FUNASRAPI void FunWfstDecoderLoadHwsRes(FUNASR_DEC_HANDLE handle, int inc_bias, unordered_map<string, int> &hws_map)
+	{
+		funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)handle;
+		if (!wfst_decoder)
+			return;
+		wfst_decoder->LoadHwsRes(inc_bias, hws_map);
+	}
+	_FUNASRAPI void FunWfstDecoderUnloadHwsRes(FUNASR_DEC_HANDLE handle)
+	{
+		funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)handle;
+		if (!wfst_decoder)
+			return;
+		wfst_decoder->UnloadHwsRes();
+	}

runtime/onnxruntime/src/itn-processor.cpp

@@ -0,0 +1,89 @@
+// Acknowledgement: this code is adapted from 
+// https://github.com/wenet-e2e/WeTextProcessing/blob/master/runtime/processor/processor.cc 
+// Retrieved in Aug 2023.
+
+// Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+//               2023 Jing Du (thuduj12@163.com)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "itn-processor.h"
+
+using fst::StringTokenType;
+
+namespace funasr {
+ITNProcessor::ITNProcessor(){};
+ITNProcessor::~ITNProcessor(){};
+
+void  ITNProcessor::InitITN(const std::string& tagger_path,
+                     const std::string& verbalizer_path, 
+                     int thread_num) {
+  try{
+    tagger_.reset(StdVectorFst::Read(tagger_path));
+    LOG(INFO) << "Successfully load model from " << tagger_path;
+    verbalizer_.reset(StdVectorFst::Read(verbalizer_path));
+    LOG(INFO) << "Successfully load model from " << verbalizer_path;
+  }catch(exception const &e){
+    LOG(ERROR) << "Error loading itn models";
+    exit(-1);
+  }
+  compiler_ = std::make_shared<StringCompiler<StdArc>>(StringTokenType::BYTE);
+  printer_ = std::make_shared<StringPrinter<StdArc>>(StringTokenType::BYTE);
+
+  if (tagger_path.find("_tn_") != tagger_path.npos) {
+    parse_type_ = ParseType::kTN;
+  } else if (tagger_path.find("_itn_") != tagger_path.npos) {
+    parse_type_ = ParseType::kITN;
+  } else {
+    LOG(FATAL) << "Invalid fst prefix, prefix should contain"
+               << " either \"_tn_\" or \"_itn_\".";
+  }
+}
+
+std::string ITNProcessor::shortest_path(const StdVectorFst& lattice) {
+  StdVectorFst shortest_path;
+  fst::ShortestPath(lattice, &shortest_path, 1, true);
+
+  std::string output;
+  printer_->operator()(shortest_path, &output);
+  return output;
+}
+
+std::string ITNProcessor::compose(const std::string& input,
+                               const StdVectorFst* fst) {
+  StdVectorFst input_fst;
+  compiler_->operator()(input, &input_fst);
+
+  StdVectorFst lattice;
+  fst::Compose(input_fst, *fst, &lattice);
+  return shortest_path(lattice);
+}
+
+std::string ITNProcessor::tag(const std::string& input) {
+  return compose(input, tagger_.get());
+}
+
+std::string ITNProcessor::verbalize(const std::string& input) {
+  if (input.empty()) {
+    return "";
+  }
+  TokenParser parser(parse_type_);
+  std::string output = parser.reorder(input);
+  return compose(output, verbalizer_.get());
+}
+
+std::string ITNProcessor::Normalize(const std::string& input) {
+  return verbalize(tag(input));
+}
+
+}  // namespace funasr

runtime/onnxruntime/src/itn-processor.h

@@ -0,0 +1,56 @@
+// Acknowledgement: this code is adapted from 
+// https://github.com/wenet-e2e/WeTextProcessing/blob/master/runtime/processor/processor.h 
+// Retrieved in Aug 2023.
+
+// Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+//               2023 Jing Du (thuduj12@163.com)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef ITN_PROCESSOR_H_
+#define ITN_PROCESSOR_H_
+
+#include "fst/fstlib.h"
+#include "precomp.h"
+#include "itn-token-parser.h"
+
+using fst::StdArc;
+using fst::StdVectorFst;
+using fst::StringCompiler;
+using fst::StringPrinter;
+
+namespace funasr {
+class ITNProcessor : public ITNModel {
+ public:
+  ITNProcessor();
+  void InitITN(const std::string &itn_tagger, const std::string &itn_verbalizer, int thread_num);
+  ~ITNProcessor();
+
+  std::string tag(const std::string& input);
+  std::string verbalize(const std::string& input);
+  std::string Normalize(const std::string& input);
+
+ private:
+  std::string shortest_path(const StdVectorFst& lattice);
+  std::string compose(const std::string& input, const StdVectorFst* fst);
+
+  ParseType parse_type_;
+  std::shared_ptr<StdVectorFst> tagger_ = nullptr;
+  std::shared_ptr<StdVectorFst> verbalizer_ = nullptr;
+  std::shared_ptr<StringCompiler<StdArc>> compiler_ = nullptr;
+  std::shared_ptr<StringPrinter<StdArc>> printer_ = nullptr;
+};
+
+}  // namespace funasr
+
+#endif  // ITN_PROCESSOR_H_

runtime/onnxruntime/src/itn-token-parser.cpp

@@ -0,0 +1,157 @@
+// Acknowledgement: this code is adapted from 
+// https://github.com/wenet-e2e/WeTextProcessing/blob/master/runtime/processor/token_parser.cc
+// Retrieved in Aug 2023.
+
+// Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+//               2023 Jing Du (thuduj12@163.com)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "itn-token-parser.h"
+#include <glog/logging.h>
+#include "utf8-string.h"
+
+namespace funasr {
+const std::string EOS = "<EOS>";
+const std::set<std::string> UTF8_WHITESPACE = {" ", "\t", "\n", "\r",
+                                               "\x0b\x0c"};
+const std::set<std::string> ASCII_LETTERS = {
+    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n",
+    "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "A", "B",
+    "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P",
+    "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "_"};
+const std::unordered_map<std::string, std::vector<std::string>> TN_ORDERS = {
+    {"date", {"year", "month", "day"}},
+    {"fraction", {"denominator", "numerator"}},
+    {"measure", {"denominator", "numerator", "value"}},
+    {"money", {"value", "currency"}},
+    {"time", {"noon", "hour", "minute", "second"}}};
+const std::unordered_map<std::string, std::vector<std::string>> ITN_ORDERS = {
+    {"date", {"year", "month", "day"}},
+    {"fraction", {"sign", "numerator", "denominator"}},
+    {"measure", {"numerator", "denominator", "value"}},
+    {"money", {"currency", "value"}},
+    {"time", {"hour", "minute", "second", "noon"}}};
+
+TokenParser::TokenParser(ParseType type) {
+  if (type == ParseType::kTN) {
+    orders = TN_ORDERS;
+  } else {
+    orders = ITN_ORDERS;
+  }
+}
+
+void TokenParser::load(const std::string& input) {
+  string2chars(input, &text);
+  CHECK_GT(text.size(), 0);
+  index = 0;
+  ch = text[0];
+}
+
+bool TokenParser::read() {
+  if (index < text.size() - 1) {
+    index += 1;
+    ch = text[index];
+    return true;
+  }
+  ch = EOS;
+  return false;
+}
+
+bool TokenParser::parse_ws() {
+  bool not_eos = ch != EOS;
+  while (not_eos && ch == " ") {
+    not_eos = read();
+  }
+  return not_eos;
+}
+
+bool TokenParser::parse_char(const std::string& exp) {
+  if (ch == exp) {
+    read();
+    return true;
+  }
+  return false;
+}
+
+bool TokenParser::parse_chars(const std::string& exp) {
+  bool ok = false;
+  std::vector<std::string> chars;
+  string2chars(exp, &chars);
+  for (const auto& x : chars) {
+    ok |= parse_char(x);
+  }
+  return ok;
+}
+
+std::string TokenParser::parse_key() {
+  CHECK_NE(ch, EOS);
+  CHECK_EQ(UTF8_WHITESPACE.count(ch), 0);
+
+  std::string key = "";
+  while (ASCII_LETTERS.count(ch) > 0) {
+    key += ch;
+    read();
+  }
+  return key;
+}
+
+std::string TokenParser::parse_value() {
+  CHECK_NE(ch, EOS);
+  bool escape = false;
+
+  std::string value = "";
+  while (ch != "\"") {
+    value += ch;
+    escape = ch == "\\" && !escape;
+    read();
+    if (escape) {
+      value += ch;
+      read();
+    }
+  }
+  return value;
+}
+
+void TokenParser::parse(const std::string& input) {
+  load(input);
+  while (parse_ws()) {
+    std::string name = parse_key();
+    parse_chars(" { ");
+
+    Token token(name);
+    while (parse_ws()) {
+      if (ch == "}") {
+        parse_char("}");
+        break;
+      }
+      std::string key = parse_key();
+      parse_chars(": \"");
+      std::string value = parse_value();
+      parse_char("\"");
+      token.append(key, value);
+    }
+    tokens.emplace_back(token);
+  }
+}
+
+std::string TokenParser::reorder(const std::string& input) {
+  parse(input);
+  std::string output = "";
+  for (auto& token : tokens) {
+    output += token.string(orders) + " ";
+  }
+  return trim(output);
+}
+
+}  // namespace funasr

runtime/onnxruntime/src/itn-token-parser.h

@@ -0,0 +1,96 @@
+// Acknowledgement: this code is adapted from 
+// https://github.com/wenet-e2e/WeTextProcessing/blob/master/runtime/processor/token_parser.h
+// Retrieved in Aug 2023.
+
+// Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+//               2023 Jing Du (thuduj12@163.com)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef ITN_TOKEN_PARSER_H_
+#define ITN_TOKEN_PARSER_H_
+
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace funasr {
+
+extern const std::string EOS;
+extern const std::set<std::string> UTF8_WHITESPACE;
+extern const std::set<std::string> ASCII_LETTERS;
+extern const std::unordered_map<std::string, std::vector<std::string>>
+    TN_ORDERS;
+extern const std::unordered_map<std::string, std::vector<std::string>>
+    ITN_ORDERS;
+
+struct Token {
+  std::string name;
+  std::vector<std::string> order;
+  std::unordered_map<std::string, std::string> members;
+
+  Token(const std::string& name) : name(name) {}
+
+  void append(const std::string& key, const std::string& value) {
+    order.emplace_back(key);
+    members[key] = value;
+  }
+
+  std::string string(
+      const std::unordered_map<std::string, std::vector<std::string>>& orders) {
+    std::string output = name + " {";
+    if (orders.count(name) > 0) {
+      order = orders.at(name);
+    }
+
+    for (const auto& key : order) {
+      if (members.count(key) == 0) {
+        continue;
+      }
+      output += " " + key + ": \"" + members[key] + "\"";
+    }
+    return output + " }";
+  }
+};
+
+enum ParseType {
+  kTN = 0x00,  // Text Normalization
+  kITN = 0x01  // Inverse Text Normalization
+};
+
+class TokenParser {
+ public:
+  TokenParser(ParseType type);
+  std::string reorder(const std::string& input);
+
+ private:
+  void load(const std::string& input);
+  bool read();
+  bool parse_ws();
+  bool parse_char(const std::string& exp);
+  bool parse_chars(const std::string& exp);
+  std::string parse_key();
+  std::string parse_value();
+  void parse(const std::string& input);
+
+  int index;
+  std::string ch;
+  std::vector<std::string> text;
+  std::vector<Token> tokens;
+  std::unordered_map<std::string, std::vector<std::string>> orders;
+};
+
+}  // funasr
+
+#endif  // ITN_TOKEN_PARSER_H_

runtime/onnxruntime/src/model.cpp

@@ -0,0 +1,60 @@
+#include "precomp.h"
+
+namespace funasr {
+Model *CreateModel(std::map<std::string, std::string>& model_path, int thread_num, ASR_TYPE type)
+{
+    // offline
+    if(type == ASR_OFFLINE){
+        string am_model_path;
+        string am_cmvn_path;
+        string am_config_path;
+        string token_path;
+
+        am_model_path = PathAppend(model_path.at(MODEL_DIR), MODEL_NAME);
+        if(model_path.find(QUANTIZE) != model_path.end() && model_path.at(QUANTIZE) == "true"){
+            am_model_path = PathAppend(model_path.at(MODEL_DIR), QUANT_MODEL_NAME);
+        }
+        am_cmvn_path = PathAppend(model_path.at(MODEL_DIR), AM_CMVN_NAME);
+        am_config_path = PathAppend(model_path.at(MODEL_DIR), AM_CONFIG_NAME);
+        token_path = PathAppend(model_path.at(MODEL_DIR), TOKEN_PATH);
+
+        Model *mm;
+        mm = new Paraformer();
+        mm->InitAsr(am_model_path, am_cmvn_path, am_config_path, token_path, thread_num);
+        return mm;
+    }else if(type == ASR_ONLINE){
+        // online
+        string en_model_path;
+        string de_model_path;
+        string am_cmvn_path;
+        string am_config_path;
+        string token_path;
+
+        en_model_path = PathAppend(model_path.at(MODEL_DIR), ENCODER_NAME);
+        de_model_path = PathAppend(model_path.at(MODEL_DIR), DECODER_NAME);
+        if(model_path.find(QUANTIZE) != model_path.end() && model_path.at(QUANTIZE) == "true"){
+            en_model_path = PathAppend(model_path.at(MODEL_DIR), QUANT_ENCODER_NAME);
+            de_model_path = PathAppend(model_path.at(MODEL_DIR), QUANT_DECODER_NAME);
+        }
+        am_cmvn_path = PathAppend(model_path.at(MODEL_DIR), AM_CMVN_NAME);
+        am_config_path = PathAppend(model_path.at(MODEL_DIR), AM_CONFIG_NAME);
+        token_path = PathAppend(model_path.at(MODEL_DIR), TOKEN_PATH);
+
+        Model *mm;
+        mm = new Paraformer();
+        mm->InitAsr(en_model_path, de_model_path, am_cmvn_path, am_config_path, token_path, thread_num);
+        return mm;
+    }else{
+        LOG(ERROR)<<"Wrong ASR_TYPE : " << type;
+        exit(-1);
+    }
+}
+
+Model *CreateModel(void* asr_handle, std::vector<int> chunk_size)
+{
+    Model* mm;
+    mm = new ParaformerOnline((Paraformer*)asr_handle, chunk_size);
+    return mm;
+}
+
+} // namespace funasr

runtime/onnxruntime/src/offline-stream.cpp

@@ -0,0 +1,160 @@
+#include "precomp.h"
+
+namespace funasr {
+OfflineStream::OfflineStream(std::map<std::string, std::string>& model_path, int thread_num, bool use_gpu, int batch_size)
+{
+    // VAD model
+    if(model_path.find(VAD_DIR) != model_path.end()){
+        string vad_model_path;
+        string vad_cmvn_path;
+        string vad_config_path;
+    
+        vad_model_path = PathAppend(model_path.at(VAD_DIR), MODEL_NAME);
+        if(model_path.find(VAD_QUANT) != model_path.end() && model_path.at(VAD_QUANT) == "true"){
+            vad_model_path = PathAppend(model_path.at(VAD_DIR), QUANT_MODEL_NAME);
+        }
+        vad_cmvn_path = PathAppend(model_path.at(VAD_DIR), VAD_CMVN_NAME);
+        vad_config_path = PathAppend(model_path.at(VAD_DIR), VAD_CONFIG_NAME);
+        if (access(vad_model_path.c_str(), F_OK) != 0 ||
+            access(vad_cmvn_path.c_str(), F_OK) != 0 ||
+            access(vad_config_path.c_str(), F_OK) != 0 )
+        {
+            LOG(INFO) << "VAD model file is not exist, skip load vad model.";
+        }else{
+            vad_handle = make_unique<FsmnVad>();
+            vad_handle->InitVad(vad_model_path, vad_cmvn_path, vad_config_path, thread_num);
+            use_vad = true;
+        }
+    }
+
+    // AM model
+    if(model_path.find(MODEL_DIR) != model_path.end()){
+        string am_model_path;
+        string am_cmvn_path;
+        string am_config_path;
+        string token_path;
+        string hw_cpu_model_path;
+        string hw_gpu_model_path;
+        string seg_dict_path;
+    
+        if(use_gpu){
+            #ifdef USE_GPU
+            asr_handle = make_unique<ParaformerTorch>();
+            asr_handle->SetBatchSize(batch_size);
+            #else
+            LOG(ERROR) <<"GPU is not supported! CPU will be used! If you want to use GPU, please add -DGPU=ON when cmake";
+            asr_handle = make_unique<Paraformer>();
+            use_gpu = false;
+            #endif
+        }else{
+            if (model_path.at(MODEL_DIR).find(MODEL_SVS) != std::string::npos)
+            {
+                asr_handle = make_unique<SenseVoiceSmall>();
+                model_type = MODEL_SVS;
+            }else{
+                asr_handle = make_unique<Paraformer>();
+            }
+        }
+
+        bool enable_hotword = false;
+        hw_cpu_model_path = PathAppend(model_path.at(MODEL_DIR), MODEL_EB_NAME);
+        hw_gpu_model_path = PathAppend(model_path.at(MODEL_DIR), TORCH_MODEL_EB_NAME);
+        seg_dict_path = PathAppend(model_path.at(MODEL_DIR), MODEL_SEG_DICT);
+        if (access(hw_cpu_model_path.c_str(), F_OK) == 0) { // if model_eb.onnx exist, hotword enabled
+          enable_hotword = true;
+          asr_handle->InitHwCompiler(hw_cpu_model_path, thread_num);
+          asr_handle->InitSegDict(seg_dict_path);
+        }
+        if (use_gpu && access(hw_gpu_model_path.c_str(), F_OK) == 0) { // if model_eb.torchscript exist, hotword enabled
+          enable_hotword = true;
+          asr_handle->InitHwCompiler(hw_gpu_model_path, thread_num);
+          asr_handle->InitSegDict(seg_dict_path);
+        }
+
+        am_model_path = PathAppend(model_path.at(MODEL_DIR), MODEL_NAME);
+        if(model_path.find(QUANTIZE) != model_path.end() && model_path.at(QUANTIZE) == "true"){
+            am_model_path = PathAppend(model_path.at(MODEL_DIR), QUANT_MODEL_NAME);
+        }
+        if(use_gpu){
+            am_model_path = PathAppend(model_path.at(MODEL_DIR), TORCH_MODEL_NAME);
+            if(model_path.find(BLADEDISC) != model_path.end() && model_path.at(BLADEDISC) == "true"){
+                am_model_path = PathAppend(model_path.at(MODEL_DIR), BLADE_MODEL_NAME);
+            }
+        }
+
+        am_cmvn_path = PathAppend(model_path.at(MODEL_DIR), AM_CMVN_NAME);
+        am_config_path = PathAppend(model_path.at(MODEL_DIR), AM_CONFIG_NAME);
+        token_path = PathAppend(model_path.at(MODEL_DIR), TOKEN_PATH);
+
+        asr_handle->InitAsr(am_model_path, am_cmvn_path, am_config_path, token_path, thread_num);
+    }
+
+    // Lm resource
+    if (model_path.find(LM_DIR) != model_path.end() && model_path.at(LM_DIR) != "") {
+        string fst_path, lm_config_path, lex_path;
+        fst_path = PathAppend(model_path.at(LM_DIR), LM_FST_RES);
+        lm_config_path = PathAppend(model_path.at(LM_DIR), LM_CONFIG_NAME);
+        lex_path = PathAppend(model_path.at(LM_DIR), LEX_PATH);
+        if (access(lex_path.c_str(), F_OK) != 0 )
+        {
+            LOG(ERROR) << "Lexicon.txt file is not exist, please use the latest version. Skip load LM model.";
+        }else{
+            asr_handle->InitLm(fst_path, lm_config_path, lex_path);
+        }
+    }
+
+    // PUNC model
+    if(model_path.find(PUNC_DIR) != model_path.end()){
+        string punc_model_path;
+        string punc_config_path;
+        string token_path;
+    
+        punc_model_path = PathAppend(model_path.at(PUNC_DIR), MODEL_NAME);
+        if(model_path.find(PUNC_QUANT) != model_path.end() && model_path.at(PUNC_QUANT) == "true"){
+            punc_model_path = PathAppend(model_path.at(PUNC_DIR), QUANT_MODEL_NAME);
+        }
+        punc_config_path = PathAppend(model_path.at(PUNC_DIR), PUNC_CONFIG_NAME);
+        token_path = PathAppend(model_path.at(PUNC_DIR), TOKEN_PATH);
+
+        if (access(punc_model_path.c_str(), F_OK) != 0 ||
+            access(punc_config_path.c_str(), F_OK) != 0 ||
+            access(token_path.c_str(), F_OK) != 0)
+        {
+            LOG(INFO) << "PUNC model file is not exist, skip load punc model.";
+        }else{
+            punc_handle = make_unique<CTTransformer>();
+            punc_handle->InitPunc(punc_model_path, punc_config_path, token_path, thread_num);
+            use_punc = true;
+        }
+    }
+#if !defined(__APPLE__)
+    // Optional: ITN, here we just support language_type=MandarinEnglish
+    if(model_path.find(ITN_DIR) != model_path.end() && model_path.at(ITN_DIR) != ""){
+        string itn_tagger_path = PathAppend(model_path.at(ITN_DIR), ITN_TAGGER_NAME);
+        string itn_verbalizer_path = PathAppend(model_path.at(ITN_DIR), ITN_VERBALIZER_NAME);
+
+        if (access(itn_tagger_path.c_str(), F_OK) != 0 ||
+            access(itn_verbalizer_path.c_str(), F_OK) != 0 )
+        {
+            LOG(INFO) << "ITN model file is not exist, skip load ITN model.";
+        }else{
+            itn_handle = make_unique<ITNProcessor>();
+            itn_handle->InitITN(itn_tagger_path, itn_verbalizer_path, thread_num);
+            use_itn = true;
+        }
+    }
+#endif
+    if(model_type == MODEL_SVS){
+        use_itn = false;
+        use_punc = false;
+    }
+}
+
+OfflineStream *CreateOfflineStream(std::map<std::string, std::string>& model_path, int thread_num, bool use_gpu, int batch_size)
+{
+    OfflineStream *mm;
+    mm = new OfflineStream(model_path, thread_num, use_gpu, batch_size);
+    return mm;
+}
+
+} // namespace funasr

runtime/onnxruntime/src/paraformer-online.cpp

@@ -0,0 +1,608 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#include "precomp.h"
+
+using namespace std;
+
+namespace funasr {
+
+ParaformerOnline::ParaformerOnline(Model* offline_handle, std::vector<int> chunk_size, std::string model_type)
+:offline_handle_(std::move(offline_handle)),chunk_size(chunk_size),session_options_{}{
+    if(model_type == MODEL_PARA){
+        Paraformer* para_handle = dynamic_cast<Paraformer*>(offline_handle_);
+        InitOnline(
+        para_handle->fbank_opts_,
+        para_handle->encoder_session_,
+        para_handle->decoder_session_,
+        para_handle->en_szInputNames_,
+        para_handle->en_szOutputNames_,
+        para_handle->de_szInputNames_,
+        para_handle->de_szOutputNames_,
+        para_handle->means_list_,
+        para_handle->vars_list_,
+        para_handle->frame_length,
+        para_handle->frame_shift,
+        para_handle->n_mels,
+        para_handle->lfr_m,
+        para_handle->lfr_n,
+        para_handle->encoder_size,
+        para_handle->fsmn_layers,
+        para_handle->fsmn_lorder,
+        para_handle->fsmn_dims,
+        para_handle->cif_threshold,
+        para_handle->tail_alphas);
+    }else if(model_type == MODEL_SVS){
+        SenseVoiceSmall* svs_handle = dynamic_cast<SenseVoiceSmall*>(offline_handle_);
+        InitOnline(
+        svs_handle->fbank_opts_,
+        svs_handle->encoder_session_,
+        svs_handle->decoder_session_,
+        svs_handle->en_szInputNames_,
+        svs_handle->en_szOutputNames_,
+        svs_handle->de_szInputNames_,
+        svs_handle->de_szOutputNames_,
+        svs_handle->means_list_,
+        svs_handle->vars_list_,
+        svs_handle->frame_length,
+        svs_handle->frame_shift,
+        svs_handle->n_mels,
+        svs_handle->lfr_m,
+        svs_handle->lfr_n,
+        svs_handle->encoder_size,
+        svs_handle->fsmn_layers,
+        svs_handle->fsmn_lorder,
+        svs_handle->fsmn_dims,
+        svs_handle->cif_threshold,
+        svs_handle->tail_alphas);
+    }
+    InitCache();
+}
+
+void ParaformerOnline::InitOnline(
+        knf::FbankOptions &fbank_opts,
+        std::shared_ptr<Ort::Session> &encoder_session,
+        std::shared_ptr<Ort::Session> &decoder_session,
+        vector<const char*> &en_szInputNames,
+        vector<const char*> &en_szOutputNames,
+        vector<const char*> &de_szInputNames,
+        vector<const char*> &de_szOutputNames,
+        vector<float> &means_list,
+        vector<float> &vars_list,
+        int frame_length_,
+        int frame_shift_,
+        int n_mels_,
+        int lfr_m_,
+        int lfr_n_,
+        int encoder_size_,
+        int fsmn_layers_,
+        int fsmn_lorder_,
+        int fsmn_dims_,
+        float cif_threshold_,
+        float tail_alphas_){
+    fbank_opts_ = fbank_opts;
+    encoder_session_ = encoder_session;
+    decoder_session_ = decoder_session;
+    en_szInputNames_ = en_szInputNames;
+    en_szOutputNames_ = en_szOutputNames;
+    de_szInputNames_ = de_szInputNames;
+    de_szOutputNames_ = de_szOutputNames;
+    means_list_ = means_list;
+    vars_list_ = vars_list;
+
+    frame_length = frame_length_;
+    frame_shift = frame_shift_;
+    n_mels = n_mels_;
+    lfr_m = lfr_m_;
+    lfr_n = lfr_n_;
+    encoder_size = encoder_size_;
+    fsmn_layers = fsmn_layers_;
+    fsmn_lorder = fsmn_lorder_;
+    fsmn_dims = fsmn_dims_;
+    cif_threshold = cif_threshold_;
+    tail_alphas = tail_alphas_;
+
+    // other vars
+    sqrt_factor = std::sqrt(encoder_size);
+    for(int i=0; i<fsmn_lorder*fsmn_dims; i++){
+        fsmn_init_cache_.emplace_back(0);
+    }
+    chunk_len = chunk_size[1]*frame_shift*lfr_n*offline_handle_->GetAsrSampleRate()/1000;
+
+    frame_sample_length_ = offline_handle_->GetAsrSampleRate() / 1000 * frame_length;
+    frame_shift_sample_length_ = offline_handle_->GetAsrSampleRate() / 1000 * frame_shift;
+
+}
+
+void ParaformerOnline::FbankKaldi(float sample_rate, std::vector<std::vector<float>> &wav_feats,
+                               std::vector<float> &waves) {
+    knf::OnlineFbank fbank(fbank_opts_);
+    // cache merge
+    waves.insert(waves.begin(), input_cache_.begin(), input_cache_.end());
+    int frame_number = ComputeFrameNum(waves.size(), frame_sample_length_, frame_shift_sample_length_);
+    // Send the audio after the last frame shift position to the cache
+    input_cache_.clear();
+    input_cache_.insert(input_cache_.begin(), waves.begin() + frame_number * frame_shift_sample_length_, waves.end());
+    if (frame_number == 0) {
+        return;
+    }
+    // Delete audio that haven't undergone fbank processing
+    waves.erase(waves.begin() + (frame_number - 1) * frame_shift_sample_length_ + frame_sample_length_, waves.end());
+
+    std::vector<float> buf(waves.size());
+    for (int32_t i = 0; i != waves.size(); ++i) {
+        buf[i] = waves[i] * 32768;
+    }
+    fbank.AcceptWaveform(sample_rate, buf.data(), buf.size());
+    int32_t frames = fbank.NumFramesReady();
+    for (int32_t i = 0; i != frames; ++i) {
+        const float *frame = fbank.GetFrame(i);
+        vector<float> frame_vector(frame, frame + fbank_opts_.mel_opts.num_bins);
+        wav_feats.emplace_back(frame_vector);
+    }
+}
+
+void ParaformerOnline::ExtractFeats(float sample_rate, vector<std::vector<float>> &wav_feats,
+                                 vector<float> &waves, bool input_finished) {
+    FbankKaldi(sample_rate, wav_feats, waves);
+    // cache deal & online lfr,cmvn
+    if (wav_feats.size() > 0) {
+        if (!reserve_waveforms_.empty()) {
+        waves.insert(waves.begin(), reserve_waveforms_.begin(), reserve_waveforms_.end());
+        }
+        if (lfr_splice_cache_.empty()) {
+            for (int i = 0; i < (lfr_m - 1) / 2; i++) {
+                lfr_splice_cache_.emplace_back(wav_feats[0]);
+            }
+        }
+        if (wav_feats.size() + lfr_splice_cache_.size() >= lfr_m) {
+            wav_feats.insert(wav_feats.begin(), lfr_splice_cache_.begin(), lfr_splice_cache_.end());
+            int frame_from_waves = (waves.size() - frame_sample_length_) / frame_shift_sample_length_ + 1;
+            int minus_frame = reserve_waveforms_.empty() ? (lfr_m - 1) / 2 : 0;
+            int lfr_splice_frame_idxs = OnlineLfrCmvn(wav_feats, input_finished);
+            int reserve_frame_idx = std::abs(lfr_splice_frame_idxs - minus_frame);
+            reserve_waveforms_.clear();
+            reserve_waveforms_.insert(reserve_waveforms_.begin(),
+                                        waves.begin() + reserve_frame_idx * frame_shift_sample_length_,
+                                        waves.begin() + frame_from_waves * frame_shift_sample_length_);
+            int sample_length = (frame_from_waves - 1) * frame_shift_sample_length_ + frame_sample_length_;
+            waves.erase(waves.begin() + sample_length, waves.end());
+        } else {
+            reserve_waveforms_.clear();
+            reserve_waveforms_.insert(reserve_waveforms_.begin(),
+                                        waves.begin() + frame_sample_length_ - frame_shift_sample_length_, waves.end());
+            lfr_splice_cache_.insert(lfr_splice_cache_.end(), wav_feats.begin(), wav_feats.end());
+        }
+    } else {
+        if (input_finished) {
+            if (!reserve_waveforms_.empty()) {
+                waves = reserve_waveforms_;
+            }
+            wav_feats = lfr_splice_cache_;
+            if(wav_feats.size() == 0){
+                LOG(ERROR) << "wav_feats's size is 0";
+            }else{
+                OnlineLfrCmvn(wav_feats, input_finished);
+            }
+        }
+    }
+    if(input_finished){
+        ResetCache();
+    }
+}
+
+int ParaformerOnline::OnlineLfrCmvn(vector<vector<float>> &wav_feats, bool input_finished) {
+    vector<vector<float>> out_feats;
+    int T = wav_feats.size();
+    int T_lrf = ceil((T - (lfr_m - 1) / 2) / (float)lfr_n);
+    int lfr_splice_frame_idxs = T_lrf;
+    vector<float> p;
+    for (int i = 0; i < T_lrf; i++) {
+        if (lfr_m <= T - i * lfr_n) {
+            for (int j = 0; j < lfr_m; j++) {
+                p.insert(p.end(), wav_feats[i * lfr_n + j].begin(), wav_feats[i * lfr_n + j].end());
+            }
+            out_feats.emplace_back(p);
+            p.clear();
+        } else {
+            if (input_finished) {
+                int num_padding = lfr_m - (T - i * lfr_n);
+                for (int j = 0; j < (wav_feats.size() - i * lfr_n); j++) {
+                    p.insert(p.end(), wav_feats[i * lfr_n + j].begin(), wav_feats[i * lfr_n + j].end());
+                }
+                for (int j = 0; j < num_padding; j++) {
+                    p.insert(p.end(), wav_feats[wav_feats.size() - 1].begin(), wav_feats[wav_feats.size() - 1].end());
+                }
+                out_feats.emplace_back(p);
+                p.clear();
+            } else {
+                lfr_splice_frame_idxs = i;
+                break;
+            }
+        }
+    }
+    lfr_splice_frame_idxs = std::min(T - 1, lfr_splice_frame_idxs * lfr_n);
+    lfr_splice_cache_.clear();
+    lfr_splice_cache_.insert(lfr_splice_cache_.begin(), wav_feats.begin() + lfr_splice_frame_idxs, wav_feats.end());
+
+    // Apply cmvn
+    for (auto &out_feat: out_feats) {
+        for (int j = 0; j < means_list_.size(); j++) {
+            out_feat[j] = (out_feat[j] + means_list_[j]) * vars_list_[j];
+        }
+    }
+    wav_feats = out_feats;
+    return lfr_splice_frame_idxs;
+}
+
+void ParaformerOnline::GetPosEmb(std::vector<std::vector<float>> &wav_feats, int timesteps, int feat_dim)
+{
+    int start_idx = start_idx_cache_;
+    start_idx_cache_ += timesteps;
+    int mm = start_idx_cache_;
+
+    int i;
+    float scale = -0.0330119726594128;
+
+    std::vector<float> tmp(mm*feat_dim);
+
+    for (i = 0; i < feat_dim/2; i++) {
+        float tmptime = exp(i * scale);
+        int j;
+        for (j = 0; j < mm; j++) {
+            int sin_idx = j * feat_dim + i;
+            int cos_idx = j * feat_dim + i + feat_dim/2;
+            float coe = tmptime * (j + 1);
+            tmp[sin_idx] = sin(coe);
+            tmp[cos_idx] = cos(coe);
+        }
+    }
+
+    for (i = start_idx; i < start_idx + timesteps; i++) {
+        for (int j = 0; j < feat_dim; j++) {
+            wav_feats[i-start_idx][j] += tmp[i*feat_dim+j];
+        }
+    }
+}
+
+void ParaformerOnline::CifSearch(std::vector<std::vector<float>> hidden, std::vector<float> alphas, bool is_final, std::vector<std::vector<float>>& list_frame)
+{
+    try{
+        int hidden_size = 0;
+        if(hidden.size() > 0){
+            hidden_size = hidden[0].size();
+        }
+        // cache
+        int i,j;
+        int chunk_size_pre = chunk_size[0];
+        for (i = 0; i < chunk_size_pre; i++)
+            alphas[i] = 0.0;
+
+        int chunk_size_suf = std::accumulate(chunk_size.begin(), chunk_size.end()-1, 0);
+        for (i = chunk_size_suf; i < alphas.size(); i++){
+            alphas[i] = 0.0;
+        }
+
+        if(hidden_cache_.size()>0){
+            hidden.insert(hidden.begin(), hidden_cache_.begin(), hidden_cache_.end());
+            alphas.insert(alphas.begin(), alphas_cache_.begin(), alphas_cache_.end());
+            hidden_cache_.clear();
+            alphas_cache_.clear();
+        }
+        
+        if (is_last_chunk) {
+            std::vector<float> tail_hidden(hidden_size, 0);
+            hidden.emplace_back(tail_hidden);
+            alphas.emplace_back(tail_alphas);
+        }
+
+        float intergrate = 0.0;
+        int len_time = alphas.size();
+        std::vector<float> frames(hidden_size, 0);
+        std::vector<float> list_fire;
+
+        for (i = 0; i < len_time; i++) {
+            float alpha = alphas[i];
+            if (alpha + intergrate < cif_threshold) {
+                intergrate += alpha;
+                list_fire.emplace_back(intergrate);
+                for (j = 0; j < hidden_size; j++) {
+                    frames[j] += alpha * hidden[i][j];
+                }
+            } else {
+                for (j = 0; j < hidden_size; j++) {
+                    frames[j] += (cif_threshold - intergrate) * hidden[i][j];
+                }
+                std::vector<float> frames_cp(frames);
+                list_frame.emplace_back(frames_cp);
+                intergrate += alpha;
+                list_fire.emplace_back(intergrate);
+                intergrate -= cif_threshold;
+                for (j = 0; j < hidden_size; j++) {
+                    frames[j] = intergrate * hidden[i][j];
+                }
+            }
+        }
+
+        // cache
+        alphas_cache_.emplace_back(intergrate);
+        if (intergrate > 0.0) {
+            std::vector<float> hidden_cache(hidden_size, 0);
+            for (i = 0; i < hidden_size; i++) {
+                hidden_cache[i] = frames[i] / intergrate;
+            }
+            hidden_cache_.emplace_back(hidden_cache);
+        } else {
+            std::vector<float> frames_cp(frames);
+            hidden_cache_.emplace_back(frames_cp);
+        }
+    }catch (std::exception const &e)
+    {
+        LOG(ERROR)<<e.what();
+    }
+}
+
+void ParaformerOnline::InitCache(){
+
+    start_idx_cache_ = 0;
+    is_first_chunk = true;
+    is_last_chunk = false;
+    hidden_cache_.clear();
+    alphas_cache_.clear();
+    feats_cache_.clear();
+    decoder_onnx.clear();
+
+    // cif cache
+    std::vector<float> hidden_cache(encoder_size, 0);
+    hidden_cache_.emplace_back(hidden_cache);
+    alphas_cache_.emplace_back(0);
+
+    // feats
+    std::vector<float> feat_cache(feat_dims, 0);
+    for(int i=0; i<(chunk_size[0]+chunk_size[2]); i++){
+        feats_cache_.emplace_back(feat_cache);
+    }
+
+    // fsmn cache
+#ifdef _WIN_X86
+    Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
+#else
+    Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
+#endif
+    const int64_t fsmn_shape_[3] = {1, fsmn_dims, fsmn_lorder};
+    for(int l=0; l<fsmn_layers; l++){
+        Ort::Value onnx_fsmn_cache = Ort::Value::CreateTensor<float>(
+            m_memoryInfo,
+            fsmn_init_cache_.data(),
+            fsmn_init_cache_.size(),
+            fsmn_shape_,
+            3);
+        decoder_onnx.emplace_back(std::move(onnx_fsmn_cache));
+    }
+};
+
+void ParaformerOnline::Reset()
+{
+    InitCache();
+}
+
+void ParaformerOnline::ResetCache() {
+    reserve_waveforms_.clear();
+    input_cache_.clear();
+    lfr_splice_cache_.clear();
+}
+
+void ParaformerOnline::AddOverlapChunk(std::vector<std::vector<float>> &wav_feats, bool input_finished){
+    wav_feats.insert(wav_feats.begin(), feats_cache_.begin(), feats_cache_.end());
+    if(input_finished){
+        feats_cache_.clear();
+        feats_cache_.insert(feats_cache_.begin(), wav_feats.end()-chunk_size[0], wav_feats.end());
+        if(!is_last_chunk){
+            int padding_length = std::accumulate(chunk_size.begin(), chunk_size.end(), 0) - wav_feats.size();
+            std::vector<float> tmp(feat_dims, 0);
+            for(int i=0; i<padding_length; i++){
+                wav_feats.emplace_back(feat_dims);
+            }
+        }
+    }else{
+        feats_cache_.clear();
+        feats_cache_.insert(feats_cache_.begin(), wav_feats.end()-chunk_size[0]-chunk_size[2], wav_feats.end());        
+    }
+}
+
+string ParaformerOnline::ForwardChunk(std::vector<std::vector<float>> &chunk_feats, bool input_finished)
+{
+    string result;
+    try{
+        int32_t num_frames = chunk_feats.size();
+
+    #ifdef _WIN_X86
+            Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
+    #else
+            Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
+    #endif
+        const int64_t input_shape_[3] = {1, num_frames, feat_dims};
+        std::vector<float> wav_feats;
+        for (const auto &chunk_feat: chunk_feats) {
+            wav_feats.insert(wav_feats.end(), chunk_feat.begin(), chunk_feat.end());
+        }
+        Ort::Value onnx_feats = Ort::Value::CreateTensor<float>(
+            m_memoryInfo,
+            wav_feats.data(),
+            wav_feats.size(),
+            input_shape_,
+            3);
+
+        const int64_t paraformer_length_shape[1] = {1};
+        std::vector<int32_t> paraformer_length;
+        paraformer_length.emplace_back(num_frames);
+        Ort::Value onnx_feats_len = Ort::Value::CreateTensor<int32_t>(
+            m_memoryInfo, paraformer_length.data(), paraformer_length.size(), paraformer_length_shape, 1);
+        
+        std::vector<Ort::Value> input_onnx;
+        input_onnx.emplace_back(std::move(onnx_feats));
+        input_onnx.emplace_back(std::move(onnx_feats_len)); 
+        
+        auto encoder_tensor = encoder_session_->Run(Ort::RunOptions{nullptr}, en_szInputNames_.data(), input_onnx.data(), input_onnx.size(), en_szOutputNames_.data(), en_szOutputNames_.size());
+
+        // get enc_vec
+        std::vector<int64_t> enc_shape = encoder_tensor[0].GetTensorTypeAndShapeInfo().GetShape();
+        float* enc_data = encoder_tensor[0].GetTensorMutableData<float>();
+        std::vector<std::vector<float>> enc_vec(enc_shape[1], std::vector<float>(enc_shape[2]));
+        for (int i = 0; i < enc_shape[1]; i++) {
+            for (int j = 0; j < enc_shape[2]; j++) {
+                enc_vec[i][j] = enc_data[i * enc_shape[2] + j];
+            }
+        }
+
+        // get alpha_vec
+        std::vector<int64_t> alpha_shape = encoder_tensor[2].GetTensorTypeAndShapeInfo().GetShape();
+        float* alpha_data = encoder_tensor[2].GetTensorMutableData<float>();
+        std::vector<float> alpha_vec(alpha_shape[1]);
+        for (int i = 0; i < alpha_shape[1]; i++) {
+            alpha_vec[i] = alpha_data[i];
+        } 
+
+        std::vector<std::vector<float>> list_frame;
+        CifSearch(enc_vec, alpha_vec, input_finished, list_frame);
+
+        
+        if(list_frame.size()>0){
+            // enc
+            decoder_onnx.insert(decoder_onnx.begin(), std::move(encoder_tensor[0]));
+            // enc_lens
+            decoder_onnx.insert(decoder_onnx.begin()+1, std::move(encoder_tensor[1]));
+
+            // acoustic_embeds
+            const int64_t emb_shape_[3] = {1, (int64_t)list_frame.size(), (int64_t)list_frame[0].size()};
+            std::vector<float> emb_input;
+            for (const auto &list_frame_: list_frame) {
+                emb_input.insert(emb_input.end(), list_frame_.begin(), list_frame_.end());
+            }
+            Ort::Value onnx_emb = Ort::Value::CreateTensor<float>(
+                m_memoryInfo,
+                emb_input.data(),
+                emb_input.size(),
+                emb_shape_,
+                3);
+            decoder_onnx.insert(decoder_onnx.begin()+2, std::move(onnx_emb));
+
+            // acoustic_embeds_len
+            const int64_t emb_length_shape[1] = {1};
+            std::vector<int32_t> emb_length;
+            emb_length.emplace_back(list_frame.size());
+            Ort::Value onnx_emb_len = Ort::Value::CreateTensor<int32_t>(
+                m_memoryInfo, emb_length.data(), emb_length.size(), emb_length_shape, 1);
+            decoder_onnx.insert(decoder_onnx.begin()+3, std::move(onnx_emb_len));
+
+            auto decoder_tensor = decoder_session_->Run(Ort::RunOptions{nullptr}, de_szInputNames_.data(), decoder_onnx.data(), decoder_onnx.size(), de_szOutputNames_.data(), de_szOutputNames_.size());
+            // fsmn cache
+            try{
+                decoder_onnx.clear();
+            }catch (std::exception const &e)
+            {
+                LOG(ERROR)<<e.what();
+                return result;
+            }
+            for(int l=0;l<fsmn_layers;l++){
+                decoder_onnx.emplace_back(std::move(decoder_tensor[2+l]));
+            }
+
+            std::vector<int64_t> decoder_shape = decoder_tensor[0].GetTensorTypeAndShapeInfo().GetShape();
+            float* float_data = decoder_tensor[0].GetTensorMutableData<float>();
+            result = offline_handle_->GreedySearch(float_data, list_frame.size(), decoder_shape[2]);
+        }
+    }catch (std::exception const &e)
+    {
+        LOG(ERROR)<<e.what();
+        return result;
+    }
+    return result;
+}
+
+string ParaformerOnline::Forward(float* din, int len, bool input_finished, const std::vector<std::vector<float>> &hw_emb, void* wfst_decoder)
+{
+    std::vector<std::vector<float>> wav_feats;
+    std::vector<float> waves(din, din+len);
+
+    string result="";
+    try{
+        if(len <16*60 && input_finished && !is_first_chunk){
+            is_last_chunk = true;
+            wav_feats = feats_cache_;
+            result = ForwardChunk(wav_feats, is_last_chunk);
+            // reset
+            ResetCache();
+            Reset();
+            return result;
+        }
+        if(is_first_chunk){
+            is_first_chunk = false;
+        }
+        ExtractFeats(offline_handle_->GetAsrSampleRate(), wav_feats, waves, input_finished);
+        if(wav_feats.size() == 0){
+            return result;
+        }
+        
+        for (auto& row : wav_feats) {
+            for (auto& val : row) {
+                val *= sqrt_factor;
+            }
+        }
+
+        GetPosEmb(wav_feats, wav_feats.size(), wav_feats[0].size());
+        if(input_finished){
+            if(wav_feats.size()+chunk_size[2] <= chunk_size[1]){
+                is_last_chunk = true;
+                AddOverlapChunk(wav_feats, input_finished);
+            }else{
+                // first chunk
+                std::vector<std::vector<float>> first_chunk;
+                first_chunk.insert(first_chunk.begin(), wav_feats.begin(), wav_feats.end());
+                AddOverlapChunk(first_chunk, input_finished);
+                string str_first_chunk = ForwardChunk(first_chunk, is_last_chunk);
+
+                // last chunk
+                is_last_chunk = true;
+                std::vector<std::vector<float>> last_chunk;
+                last_chunk.insert(last_chunk.begin(), wav_feats.end()-(wav_feats.size()+chunk_size[2]-chunk_size[1]), wav_feats.end());
+                AddOverlapChunk(last_chunk, input_finished);
+                string str_last_chunk = ForwardChunk(last_chunk, is_last_chunk);
+
+                result = str_first_chunk+str_last_chunk;
+                // reset
+                ResetCache();
+                Reset();
+                return result;
+            }
+        }else{
+            AddOverlapChunk(wav_feats, input_finished);
+        }
+
+        result = ForwardChunk(wav_feats, is_last_chunk);
+        if(input_finished){
+            // reset
+            ResetCache();
+            Reset();
+        }
+    }catch (std::exception const &e)
+    {
+        LOG(ERROR)<<e.what();
+        return result;
+    }
+
+    return result;
+}
+
+ParaformerOnline::~ParaformerOnline()
+{
+}
+
+string ParaformerOnline::Rescoring()
+{
+    LOG(ERROR)<<"Not Imp!!!!!!";
+    return "";
+}
+} // namespace funasr

runtime/onnxruntime/src/paraformer-online.h

@@ -0,0 +1,133 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+#pragma once
+
+#include "precomp.h"
+
+namespace funasr {
+
+    class ParaformerOnline : public Model {
+    /**
+     * Author: Speech Lab of DAMO Academy, Alibaba Group
+     * ParaformerOnline: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition
+     * https://arxiv.org/pdf/2206.08317.pdf
+    */
+    private:
+
+        void FbankKaldi(float sample_rate, std::vector<std::vector<float>> &wav_feats,
+                std::vector<float> &waves);
+        int OnlineLfrCmvn(vector<vector<float>> &wav_feats, bool input_finished);
+        void GetPosEmb(std::vector<std::vector<float>> &wav_feats, int timesteps, int feat_dim);
+        void CifSearch(std::vector<std::vector<float>> hidden, std::vector<float> alphas, bool is_final, std::vector<std::vector<float>> &list_frame);
+
+        static int ComputeFrameNum(int sample_length, int frame_sample_length, int frame_shift_sample_length) {
+            int frame_num = static_cast<int>((sample_length - frame_sample_length) / frame_shift_sample_length + 1);
+            if (frame_num >= 1 && sample_length >= frame_sample_length)
+                return frame_num;
+            else
+                return 0;
+        }
+        void InitOnline(
+            knf::FbankOptions &fbank_opts,
+            std::shared_ptr<Ort::Session> &encoder_session,
+            std::shared_ptr<Ort::Session> &decoder_session,
+            vector<const char*> &en_szInputNames,
+            vector<const char*> &en_szOutputNames,
+            vector<const char*> &de_szInputNames,
+            vector<const char*> &de_szOutputNames,
+            vector<float> &means_list,
+            vector<float> &vars_list,
+            int frame_length_,
+            int frame_shift_,
+            int n_mels_,
+            int lfr_m_,
+            int lfr_n_,
+            int encoder_size_,
+            int fsmn_layers_,
+            int fsmn_lorder_,
+            int fsmn_dims_,
+            float cif_threshold_,
+            float tail_alphas_);
+
+        void StartUtterance()
+        {
+        }
+        
+        void EndUtterance()
+        {
+        }
+        
+        Model* offline_handle_ = nullptr;
+        // from offline_handle_
+        knf::FbankOptions fbank_opts_;
+        std::shared_ptr<Ort::Session> encoder_session_ = nullptr;
+        std::shared_ptr<Ort::Session> decoder_session_ = nullptr;
+        Ort::SessionOptions session_options_;
+        vector<const char*> en_szInputNames_;
+        vector<const char*> en_szOutputNames_;
+        vector<const char*> de_szInputNames_;
+        vector<const char*> de_szOutputNames_;
+        vector<float> means_list_;
+        vector<float> vars_list_;
+        // configs from offline_handle_
+        int frame_length = 25;
+        int frame_shift = 10;
+        int n_mels = 80;
+        int lfr_m = PARA_LFR_M;
+        int lfr_n = PARA_LFR_N;
+        int encoder_size = 512;
+        int fsmn_layers = 16;
+        int fsmn_lorder = 10;
+        int fsmn_dims = 512;
+        float cif_threshold = 1.0;
+        float tail_alphas = 0.45;
+
+        // configs
+        int feat_dims = lfr_m*n_mels;
+        std::vector<int> chunk_size = {5,10,5};        
+        int frame_sample_length_ = MODEL_SAMPLE_RATE / 1000 * frame_length;
+        int frame_shift_sample_length_ = MODEL_SAMPLE_RATE / 1000 * frame_shift;
+
+        // The reserved waveforms by fbank
+        std::vector<float> reserve_waveforms_;
+        // waveforms reserved after last shift position
+        std::vector<float> input_cache_;
+        // lfr reserved cache
+        std::vector<std::vector<float>> lfr_splice_cache_;
+        // position index cache
+        int start_idx_cache_ = 0;
+        // cif alpha
+        std::vector<float> alphas_cache_;
+        std::vector<std::vector<float>> hidden_cache_;
+        std::vector<std::vector<float>> feats_cache_;
+        // fsmn init caches
+        std::vector<float> fsmn_init_cache_;
+        std::vector<Ort::Value> decoder_onnx;
+
+        bool is_first_chunk = true;
+        bool is_last_chunk = false;
+        double sqrt_factor;
+
+    public:
+        ParaformerOnline(Model* offline_handle, std::vector<int> chunk_size, std::string model_type=MODEL_PARA);
+        ~ParaformerOnline();
+        void Reset();
+        void ResetCache();
+        void InitCache();
+        void ExtractFeats(float sample_rate, vector<vector<float>> &wav_feats, vector<float> &waves, bool input_finished);
+        void AddOverlapChunk(std::vector<std::vector<float>> &wav_feats, bool input_finished);
+        
+        string ForwardChunk(std::vector<std::vector<float>> &wav_feats, bool input_finished);
+        string Forward(float* din, int len, bool input_finished, const std::vector<std::vector<float>> &hw_emb={{0.0}}, void* wfst_decoder=nullptr);
+        string Rescoring();
+
+        int GetAsrSampleRate() { return offline_handle_->GetAsrSampleRate(); };
+
+        // 2pass
+        std::string online_res;
+        int chunk_len;
+    };
+
+} // namespace funasr

runtime/onnxruntime/src/paraformer-torch.cpp

@@ -0,0 +1,638 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#include "precomp.h"
+#include "paraformer-torch.h"
+#include "encode_converter.h"
+#include <cstddef>
+
+using namespace std;
+namespace funasr {
+
+ParaformerTorch::ParaformerTorch()
+:use_hotword(false){
+}
+
+// offline
+void ParaformerTorch::InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){
+    LoadConfigFromYaml(am_config.c_str());
+    // knf options
+    fbank_opts_.frame_opts.dither = 0;
+    fbank_opts_.mel_opts.num_bins = n_mels;
+    fbank_opts_.frame_opts.samp_freq = asr_sample_rate;
+    fbank_opts_.frame_opts.window_type = window_type;
+    fbank_opts_.frame_opts.frame_shift_ms = frame_shift;
+    fbank_opts_.frame_opts.frame_length_ms = frame_length;
+    fbank_opts_.energy_floor = 0;
+    fbank_opts_.mel_opts.debug_mel = false;
+
+    vocab = new Vocab(token_file.c_str());
+	phone_set_ = new PhoneSet(token_file.c_str());
+    LoadCmvn(am_cmvn.c_str());
+
+    torch::DeviceType device = at::kCPU;
+    #ifdef USE_GPU
+    if (!torch::cuda::is_available()) {
+        LOG(ERROR) << "CUDA is not available! Please check your GPU settings";
+        exit(-1);
+    } else {
+        LOG(INFO) << "CUDA is available, running on GPU";
+        device = at::kCUDA;
+    }
+    #endif
+    #ifdef USE_IPEX
+    torch::jit::setTensorExprFuserEnabled(false);
+    #endif
+
+    try {
+        torch::jit::script::Module model = torch::jit::load(am_model, device);
+        model_ = std::make_shared<TorchModule>(std::move(model)); 
+        LOG(INFO) << "Successfully load model from " << am_model;
+        torch::NoGradGuard no_grad;
+        model_->eval();
+        torch::jit::setGraphExecutorOptimize(false);
+        torch::jit::FusionStrategy static0 = {{torch::jit::FusionBehavior::STATIC, 0}};
+        torch::jit::setFusionStrategy(static0);
+        #ifdef USE_GPU
+        WarmUp();
+        #endif
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load am model: " << am_model << e.what();
+        exit(-1);
+    }
+}
+
+void ParaformerTorch::InitLm(const std::string &lm_file, 
+                        const std::string &lm_cfg_file, 
+                        const std::string &lex_file) {
+    try {
+        lm_ = std::shared_ptr<fst::Fst<fst::StdArc>>(
+            fst::Fst<fst::StdArc>::Read(lm_file));
+        if (lm_){
+            lm_vocab = new Vocab(lm_cfg_file.c_str(), lex_file.c_str());
+            LOG(INFO) << "Successfully load lm file " << lm_file;
+        }else{
+            LOG(ERROR) << "Failed to load lm file " << lm_file;
+        }
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load lm file: " << e.what();
+        exit(0);
+    }
+}
+
+void ParaformerTorch::LoadConfigFromYaml(const char* filename){
+
+    YAML::Node config;
+    try{
+        config = YAML::LoadFile(filename);
+    }catch(exception const &e){
+        LOG(ERROR) << "Error loading file, yaml file error or not exist.";
+        exit(-1);
+    }
+
+    try{
+        YAML::Node frontend_conf = config["frontend_conf"];
+        this->asr_sample_rate = frontend_conf["fs"].as<int>();
+
+        YAML::Node lang_conf = config["lang"];
+        if (lang_conf.IsDefined()){
+            language = lang_conf.as<string>();
+        }
+    }catch(exception const &e){
+        LOG(ERROR) << "Error when load argument from vad config YAML.";
+        exit(-1);
+    }
+}
+
+void ParaformerTorch::InitHwCompiler(const std::string &hw_model, int thread_num) {
+    // TODO
+    torch::DeviceType device = at::kCPU;
+    #ifdef USE_GPU
+    if (!torch::cuda::is_available()) {
+        // LOG(ERROR) << "CUDA is not available! Please check your GPU settings";
+        exit(-1);
+    } else {
+        // LOG(INFO) << "CUDA is available, running on GPU";
+        device = at::kCUDA;
+    }
+    #endif
+
+    try {
+        torch::jit::script::Module model = torch::jit::load(hw_model, device);
+        hw_model_ = std::make_shared<TorchModule>(std::move(model));
+        LOG(INFO) << "Successfully load model from " << hw_model;
+        torch::NoGradGuard no_grad;
+        hw_model_->eval();
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load hw model: " << hw_model << e.what();
+        exit(-1);
+    }
+    use_hotword = true;
+}
+
+void ParaformerTorch::InitSegDict(const std::string &seg_dict_model) {
+    seg_dict = new SegDict(seg_dict_model.c_str());
+}
+
+ParaformerTorch::~ParaformerTorch()
+{
+    if(vocab){
+        delete vocab;
+        vocab = nullptr;
+    }
+    if(lm_vocab){
+        delete lm_vocab;
+        lm_vocab = nullptr;
+    }
+    if(seg_dict){
+        delete seg_dict;
+        seg_dict = nullptr;
+    }
+    if(phone_set_){
+        delete phone_set_;
+        phone_set_ = nullptr;
+    }
+}
+
+void ParaformerTorch::StartUtterance()
+{
+}
+
+void ParaformerTorch::EndUtterance()
+{
+}
+
+void ParaformerTorch::Reset()
+{
+}
+
+void ParaformerTorch::FbankKaldi(float sample_rate, const float* waves, int len, std::vector<std::vector<float>> &asr_feats) {
+    knf::OnlineFbank fbank_(fbank_opts_);
+    std::vector<float> buf(len);
+    for (int32_t i = 0; i != len; ++i) {
+        buf[i] = waves[i] * 32768;
+    }
+    fbank_.AcceptWaveform(sample_rate, buf.data(), buf.size());
+
+    int32_t frames = fbank_.NumFramesReady();
+    for (int32_t i = 0; i != frames; ++i) {
+        const float *frame = fbank_.GetFrame(i);
+        std::vector<float> frame_vector(frame, frame + fbank_opts_.mel_opts.num_bins);
+        asr_feats.emplace_back(frame_vector);
+    }
+}
+
+void ParaformerTorch::LoadCmvn(const char *filename)
+{
+    ifstream cmvn_stream(filename);
+    if (!cmvn_stream.is_open()) {
+        LOG(ERROR) << "Failed to open file: " << filename;
+        exit(-1);
+    }
+    string line;
+
+    while (getline(cmvn_stream, line)) {
+        istringstream iss(line);
+        vector<string> line_item{istream_iterator<string>{iss}, istream_iterator<string>{}};
+        if (line_item[0] == "<AddShift>") {
+            getline(cmvn_stream, line);
+            istringstream means_lines_stream(line);
+            vector<string> means_lines{istream_iterator<string>{means_lines_stream}, istream_iterator<string>{}};
+            if (means_lines[0] == "<LearnRateCoef>") {
+                for (int j = 3; j < means_lines.size() - 1; j++) {
+                    means_list_.push_back(stof(means_lines[j]));
+                }
+                continue;
+            }
+        }
+        else if (line_item[0] == "<Rescale>") {
+            getline(cmvn_stream, line);
+            istringstream vars_lines_stream(line);
+            vector<string> vars_lines{istream_iterator<string>{vars_lines_stream}, istream_iterator<string>{}};
+            if (vars_lines[0] == "<LearnRateCoef>") {
+                for (int j = 3; j < vars_lines.size() - 1; j++) {
+                    vars_list_.push_back(stof(vars_lines[j])*scale);
+                }
+                continue;
+            }
+        }
+    }
+}
+
+string ParaformerTorch::GreedySearch(float * in, int n_len,  int64_t token_nums, bool is_stamp, std::vector<float> us_alphas, std::vector<float> us_cif_peak)
+{
+    vector<int> hyps;
+    int Tmax = n_len;
+    for (int i = 0; i < Tmax; i++) {
+        int max_idx;
+        float max_val;
+        FindMax(in + i * token_nums, token_nums, max_val, max_idx);
+        hyps.push_back(max_idx);
+    }
+    if(!is_stamp){
+        return vocab->Vector2StringV2(hyps, language);
+    }else{
+        std::vector<string> char_list;
+        std::vector<std::vector<float>> timestamp_list;
+        std::string res_str;
+        vocab->Vector2String(hyps, char_list);
+        std::vector<string> raw_char(char_list);
+        TimestampOnnx(us_alphas, us_cif_peak, char_list, res_str, timestamp_list);
+
+        return PostProcess(raw_char, timestamp_list);
+    }
+}
+
+string ParaformerTorch::BeamSearch(WfstDecoder* &wfst_decoder, float *in, int len, int64_t token_nums)
+{
+  return wfst_decoder->Search(in, len, token_nums);
+}
+
+string ParaformerTorch::FinalizeDecode(WfstDecoder* &wfst_decoder,
+                                  bool is_stamp, std::vector<float> us_alphas, std::vector<float> us_cif_peak)
+{
+  return wfst_decoder->FinalizeDecode(is_stamp, us_alphas, us_cif_peak);
+}
+
+void ParaformerTorch::LfrCmvn(std::vector<std::vector<float>> &asr_feats) {
+
+    std::vector<std::vector<float>> out_feats;
+    int T = asr_feats.size();
+    int T_lrf = ceil(1.0 * T / lfr_n);
+
+    // Pad frames at start(copy first frame)
+    for (int i = 0; i < (lfr_m - 1) / 2; i++) {
+        asr_feats.insert(asr_feats.begin(), asr_feats[0]);
+    }
+    // Merge lfr_m frames as one,lfr_n frames per window
+    T = T + (lfr_m - 1) / 2;
+    std::vector<float> p;
+    for (int i = 0; i < T_lrf; i++) {
+        if (lfr_m <= T - i * lfr_n) {
+            for (int j = 0; j < lfr_m; j++) {
+                p.insert(p.end(), asr_feats[i * lfr_n + j].begin(), asr_feats[i * lfr_n + j].end());
+            }
+            out_feats.emplace_back(p);
+            p.clear();
+        } else {
+            // Fill to lfr_m frames at last window if less than lfr_m frames  (copy last frame)
+            int num_padding = lfr_m - (T - i * lfr_n);
+            for (int j = 0; j < (asr_feats.size() - i * lfr_n); j++) {
+                p.insert(p.end(), asr_feats[i * lfr_n + j].begin(), asr_feats[i * lfr_n + j].end());
+            }
+            for (int j = 0; j < num_padding; j++) {
+                p.insert(p.end(), asr_feats[asr_feats.size() - 1].begin(), asr_feats[asr_feats.size() - 1].end());
+            }
+            out_feats.emplace_back(p);
+            p.clear();
+        }
+    }
+    // Apply cmvn
+    for (auto &out_feat: out_feats) {
+        for (int j = 0; j < means_list_.size(); j++) {
+            out_feat[j] = (out_feat[j] + means_list_[j]) * vars_list_[j];
+        }
+    }
+    asr_feats = out_feats;
+}
+
+std::vector<std::string> ParaformerTorch::Forward(float** din, int* len, bool input_finished, const std::vector<std::vector<float>> &hw_emb, void* decoder_handle, int batch_in)
+{
+    vector<std::string> results;
+    string result="";
+
+    WfstDecoder* wfst_decoder = (WfstDecoder*)decoder_handle;
+    int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins;
+    int32_t feature_dim = lfr_m*in_feat_dim;
+
+    std::vector<vector<float>> feats_batch;
+    std::vector<int32_t> paraformer_length;
+    int max_size = 0;
+    int max_frames = 0;
+    for(int index=0; index<batch_in; index++){
+        std::vector<std::vector<float>> asr_feats;
+        FbankKaldi(asr_sample_rate, din[index], len[index], asr_feats);
+        if(asr_feats.size() != 0){
+            LfrCmvn(asr_feats);
+        }
+        int32_t num_frames  = asr_feats.size();
+        paraformer_length.emplace_back(num_frames);
+        if(max_size < asr_feats.size()*feature_dim){
+            max_size = asr_feats.size()*feature_dim;
+            max_frames = num_frames;
+        }
+
+        std::vector<float> flattened;
+        for (const auto& sub_vector : asr_feats) {
+            flattened.insert(flattened.end(), sub_vector.begin(), sub_vector.end());
+        }
+        feats_batch.emplace_back(flattened);
+    }
+
+    if(max_frames == 0){
+        for(int index=0; index<batch_in; index++){
+            results.push_back(result);
+        }
+        return results;
+    }
+
+    // padding
+    std::vector<float> all_feats(batch_in * max_frames * feature_dim);
+    for(int index=0; index<batch_in; index++){
+        feats_batch[index].resize(max_size);
+        std::memcpy(&all_feats[index * max_frames * feature_dim], feats_batch[index].data(),
+                        max_frames * feature_dim * sizeof(float));
+    }
+    torch::Tensor feats =
+        torch::from_blob(all_feats.data(),
+                {batch_in, max_frames, feature_dim}, torch::kFloat).contiguous();
+    torch::Tensor feat_lens = torch::from_blob(paraformer_length.data(),
+                        {batch_in}, torch::kInt32);
+
+    // 2. forward
+    #ifdef USE_GPU
+    feats = feats.to(at::kCUDA);
+    feat_lens = feat_lens.to(at::kCUDA);
+    #endif
+    std::vector<torch::jit::IValue> inputs = {feats, feat_lens};
+
+    std::vector<float> batch_embedding;
+    std::vector<float> embedding;
+    try{
+        if (use_hotword) {
+            if(hw_emb.size()<=0){
+                LOG(ERROR) << "hw_emb is null";
+                for(int index=0; index<batch_in; index++){
+                    results.push_back(result);
+                }
+                return results;
+            }
+            
+            embedding.reserve(hw_emb.size() * hw_emb[0].size());
+            for (auto item : hw_emb) {
+                embedding.insert(embedding.end(), item.begin(), item.end());
+            }
+            batch_embedding.reserve(batch_in * embedding.size());
+            for (size_t index = 0; index < batch_in; ++index) {
+                batch_embedding.insert(batch_embedding.end(), embedding.begin(), embedding.end());
+            }
+
+            torch::Tensor tensor_hw_emb =
+                torch::from_blob(batch_embedding.data(),
+                        {batch_in, static_cast<int64_t>(hw_emb.size()), static_cast<int64_t>(hw_emb[0].size())}, torch::kFloat).contiguous();
+            #ifdef USE_GPU
+            tensor_hw_emb = tensor_hw_emb.to(at::kCUDA);
+            #endif
+            inputs.emplace_back(tensor_hw_emb);
+        }
+    }catch (std::exception const &e)
+    {
+        LOG(ERROR)<<e.what();
+        for(int index=0; index<batch_in; index++){
+            results.push_back(result);
+        }
+        return results;
+    }
+
+    try {
+        if(inputs.size() == 0){
+            LOG(ERROR) << "inputs of forward is null";
+            for(int index=0; index<batch_in; index++){
+                results.push_back(result);
+            }
+            return results;
+        }
+        auto outputs = model_->forward(inputs).toTuple()->elements();
+        torch::Tensor am_scores;
+        torch::Tensor valid_token_lens;
+        #ifdef USE_GPU
+        am_scores = outputs[0].toTensor().to(at::kCPU);
+        valid_token_lens = outputs[1].toTensor().to(at::kCPU);
+        #else
+        am_scores = outputs[0].toTensor();
+        valid_token_lens = outputs[1].toTensor();
+        #endif
+
+        torch::Tensor us_alphas_tensor;
+        torch::Tensor us_peaks_tensor;
+        if(outputs.size() == 4){
+            #ifdef USE_GPU
+            us_alphas_tensor = outputs[2].toTensor().to(at::kCPU);
+            us_peaks_tensor = outputs[3].toTensor().to(at::kCPU);
+            #else
+            us_alphas_tensor = outputs[2].toTensor();
+            us_peaks_tensor = outputs[3].toTensor();
+            #endif
+        }
+
+        // timestamp
+        for(int index=0; index<batch_in; index++){
+            result="";
+            if(outputs.size() == 4){
+                float* us_alphas_data = us_alphas_tensor[index].data_ptr<float>();
+                std::vector<float> us_alphas(paraformer_length[index]*3);
+                for (int i = 0; i < us_alphas.size(); i++) {
+                    us_alphas[i] = us_alphas_data[i];
+                }
+
+                float* us_peaks_data = us_peaks_tensor[index].data_ptr<float>();
+                std::vector<float> us_peaks(paraformer_length[index]*3);
+                for (int i = 0; i < us_peaks.size(); i++) {
+                    us_peaks[i] = us_peaks_data[i];
+                }
+                if (lm_ == nullptr) {
+                    result = GreedySearch(am_scores[index].data_ptr<float>(), valid_token_lens[index].item<int>(), am_scores.size(2), true, us_alphas, us_peaks);
+                } else {
+                    result = BeamSearch(wfst_decoder, am_scores[index].data_ptr<float>(), valid_token_lens[index].item<int>(), am_scores.size(2));
+                    if (input_finished) {
+                        result = FinalizeDecode(wfst_decoder, true, us_alphas, us_peaks);
+                    }
+                }
+            }else{
+                if (lm_ == nullptr) {
+                    result = GreedySearch(am_scores[index].data_ptr<float>(), valid_token_lens[index].item<int>(), am_scores.size(2));
+                } else {
+                    result = BeamSearch(wfst_decoder, am_scores[index].data_ptr<float>(), valid_token_lens[index].item<int>(), am_scores.size(2));
+                    if (input_finished) {
+                        result = FinalizeDecode(wfst_decoder);
+                    }
+                }
+            }
+            results.push_back(result);
+			if (wfst_decoder){
+				wfst_decoder->StartUtterance();
+			}
+        }
+    }
+    catch (std::exception const &e)
+    {
+        LOG(ERROR)<<e.what();
+    }
+
+    return results;
+}
+
+void ParaformerTorch::WarmUp()
+{
+    int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins;
+    int32_t feature_dim = lfr_m*in_feat_dim;
+    int batch_in = 1;
+    int max_frames = 10;
+    std::vector<int32_t> paraformer_length;
+    paraformer_length.push_back(max_frames);
+
+    std::vector<float> all_feats(batch_in * max_frames * feature_dim, 0.1);
+    torch::Tensor feats =
+        torch::from_blob(all_feats.data(),
+                {batch_in, max_frames, feature_dim}, torch::kFloat).contiguous();
+    torch::Tensor feat_lens = torch::from_blob(paraformer_length.data(),
+                        {batch_in}, torch::kInt32);
+
+    // 2. forward
+    feats = feats.to(at::kCUDA);
+    feat_lens = feat_lens.to(at::kCUDA);
+    std::vector<torch::jit::IValue> inputs = {feats, feat_lens};
+
+    if (use_hotword) {
+        std::string hotwords_wp = "";
+        std::vector<std::vector<float>> hw_emb =  CompileHotwordEmbedding(hotwords_wp);
+        std::vector<float> embedding;
+        embedding.reserve(hw_emb.size() * hw_emb[0].size());
+        for (auto item : hw_emb) {
+            embedding.insert(embedding.end(), item.begin(), item.end());
+        }
+        torch::Tensor tensor_hw_emb =
+            torch::from_blob(embedding.data(),
+                    {batch_in, static_cast<int64_t>(hw_emb.size()), static_cast<int64_t>(hw_emb[0].size())}, torch::kFloat).contiguous();
+        tensor_hw_emb = tensor_hw_emb.to(at::kCUDA);
+        inputs.emplace_back(tensor_hw_emb);
+    }
+
+    try {
+        auto outputs = model_->forward(inputs).toTuple()->elements();
+    }
+    catch (std::exception const &e)
+    {
+        LOG(ERROR)<<e.what();
+    }
+}
+
+std::vector<std::vector<float>> ParaformerTorch::CompileHotwordEmbedding(std::string &hotwords) {
+    int embedding_dim = encoder_size;
+    std::vector<std::vector<float>> hw_emb;
+    if (!use_hotword) {
+        std::vector<float> vec(embedding_dim, 0);
+        hw_emb.push_back(vec);
+        return hw_emb;
+    }
+    int max_hotword_len = 10;
+    std::vector<int32_t> hotword_matrix;
+    std::vector<int32_t> lengths;
+    int hotword_size = 1;
+    int real_hw_size = 0;
+    if (!hotwords.empty()) {
+      std::vector<std::string> hotword_array = split(hotwords, ' ');
+      hotword_size = hotword_array.size() + 1;
+      hotword_matrix.reserve(hotword_size * max_hotword_len);
+      for (auto hotword : hotword_array) {
+        std::vector<std::string> chars;
+        if (EncodeConverter::IsAllChineseCharactor((const U8CHAR_T*)hotword.c_str(), hotword.size())) {
+          KeepChineseCharacterAndSplit(hotword, chars);
+        } else {
+          // for english
+          std::vector<std::string> words = split(hotword, ' ');
+          for (auto word : words) {
+            std::vector<string> tokens = seg_dict->GetTokensByWord(word);
+            chars.insert(chars.end(), tokens.begin(), tokens.end());
+          }
+        }
+        if(chars.size()==0){
+            continue;
+        }
+        std::vector<int32_t> hw_vector(max_hotword_len, 0);
+        int vector_len = std::min(max_hotword_len, (int)chars.size());
+        int chs_oov = false;
+        for (int i=0; i<vector_len; i++) {
+          hw_vector[i] = phone_set_->String2Id(chars[i]);
+          if(hw_vector[i] == -1){
+            chs_oov = true;
+            break;
+          }
+        }
+        if(chs_oov){
+          LOG(INFO) << "OOV: " << hotword;
+          continue;
+        }
+        LOG(INFO) << hotword;
+        lengths.push_back(vector_len);
+        real_hw_size += 1;
+        hotword_matrix.insert(hotword_matrix.end(), hw_vector.begin(), hw_vector.end());
+      }
+      hotword_size = real_hw_size + 1;
+    }
+    std::vector<int32_t> blank_vec(max_hotword_len, 0);
+    blank_vec[0] = 1;
+    hotword_matrix.insert(hotword_matrix.end(), blank_vec.begin(), blank_vec.end());
+    lengths.push_back(1);
+
+    torch::Tensor feats =
+        torch::from_blob(hotword_matrix.data(),
+                {hotword_size, max_hotword_len}, torch::kInt32).contiguous();
+
+    // 2. forward
+    #ifdef USE_GPU
+    feats = feats.to(at::kCUDA);
+    #endif
+    std::vector<torch::jit::IValue> inputs = {feats};
+    std::vector<std::vector<float>> result;
+    try {
+        auto output = hw_model_->forward(inputs);
+        torch::Tensor emb_tensor;
+        #ifdef USE_GPU
+        emb_tensor = output.toTensor().to(at::kCPU);
+        #else
+        emb_tensor = output.toTensor();
+        #endif
+        assert(emb_tensor.size(0) == max_hotword_len);
+        assert(emb_tensor.size(1) == hotword_size);
+        embedding_dim = emb_tensor.size(2);
+
+        float* floatData = emb_tensor.data_ptr<float>();
+        for (int j = 0; j < hotword_size; j++)
+        {
+            int start_pos = hotword_size * (lengths[j] - 1) * embedding_dim + j * embedding_dim;
+            std::vector<float> embedding;
+            embedding.insert(embedding.begin(), floatData + start_pos, floatData + start_pos + embedding_dim);
+            result.push_back(embedding);
+        }
+    }
+    catch (std::exception const &e)
+    {
+        LOG(ERROR)<<e.what();
+    }
+    return result;
+}
+
+Vocab* ParaformerTorch::GetVocab()
+{
+    return vocab;
+}
+
+Vocab* ParaformerTorch::GetLmVocab()
+{
+    return lm_vocab;
+}
+
+PhoneSet* ParaformerTorch::GetPhoneSet()
+{
+    return phone_set_;
+}
+
+string ParaformerTorch::Rescoring()
+{
+    LOG(ERROR)<<"Not Imp!!!!!!";
+    return "";
+}
+} // namespace funasr

runtime/onnxruntime/src/paraformer-torch.h

@@ -0,0 +1,98 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+#pragma once
+#define C10_USE_GLOG
+#include <torch/serialize.h>
+#include <torch/script.h>
+#include <torch/torch.h>
+#include <torch/csrc/jit/passes/tensorexpr_fuser.h>
+#include "precomp.h"
+#include "fst/fstlib.h"
+#include "fst/symbol-table.h"
+#include "bias-lm.h"
+#include "phone-set.h"
+
+namespace funasr {
+
+    class ParaformerTorch : public Model {
+    /**
+     * Author: Speech Lab of DAMO Academy, Alibaba Group
+     * Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition
+     * https://arxiv.org/pdf/2206.08317.pdf
+    */
+    private:
+        Vocab* vocab = nullptr;
+        Vocab* lm_vocab = nullptr;
+        SegDict* seg_dict = nullptr;
+        PhoneSet* phone_set_ = nullptr;
+        //const float scale = 22.6274169979695;
+        const float scale = 1.0;
+
+        void LoadConfigFromYaml(const char* filename);
+        void LoadCmvn(const char *filename);
+        void LfrCmvn(std::vector<std::vector<float>> &asr_feats);
+
+        using TorchModule = torch::jit::script::Module;
+        std::shared_ptr<TorchModule> model_ = nullptr;
+        std::shared_ptr<TorchModule> hw_model_ = nullptr;
+        std::vector<torch::Tensor> encoder_outs_;
+        bool use_hotword;
+
+    public:
+        ParaformerTorch();
+        ~ParaformerTorch();
+        void InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);
+        void InitHwCompiler(const std::string &hw_model, int thread_num);
+        void InitSegDict(const std::string &seg_dict_model);
+        std::vector<std::vector<float>> CompileHotwordEmbedding(std::string &hotwords);
+        void Reset();
+        void FbankKaldi(float sample_rate, const float* waves, int len, std::vector<std::vector<float>> &asr_feats);
+        void WarmUp();
+        std::vector<std::string> Forward(float** din, int* len, bool input_finished=true, const std::vector<std::vector<float>> &hw_emb={{0.0}}, void* wfst_decoder=nullptr, int batch_in=1);
+        string GreedySearch( float* in, int n_len, int64_t token_nums,
+                             bool is_stamp=false, std::vector<float> us_alphas={0}, std::vector<float> us_cif_peak={0});
+
+        string Rescoring();
+        string GetLang(){return language;};
+        int GetAsrSampleRate() { return asr_sample_rate; };
+        void SetBatchSize(int batch_size) {batch_size_ = batch_size;};
+        int GetBatchSize() {return batch_size_;};
+        void StartUtterance();
+        void EndUtterance();
+        void InitLm(const std::string &lm_file, const std::string &lm_cfg_file, const std::string &lex_file);
+        string BeamSearch(WfstDecoder* &wfst_decoder, float* in, int n_len, int64_t token_nums);
+        string FinalizeDecode(WfstDecoder* &wfst_decoder,
+                          bool is_stamp=false, std::vector<float> us_alphas={0}, std::vector<float> us_cif_peak={0});
+        Vocab* GetVocab();
+        Vocab* GetLmVocab();
+        PhoneSet* GetPhoneSet();
+		
+        knf::FbankOptions fbank_opts_;
+        vector<float> means_list_;
+        vector<float> vars_list_;
+        int lfr_m = PARA_LFR_M;
+        int lfr_n = PARA_LFR_N;
+
+        // paraformer-offline
+        std::string language="zh-cn";
+
+        // lm
+        std::shared_ptr<fst::Fst<fst::StdArc>> lm_ = nullptr;
+
+        string window_type = "hamming";
+        int frame_length = 25;
+        int frame_shift = 10;
+        int n_mels = 80;
+        int encoder_size = 512;
+        int fsmn_layers = 16;
+        int fsmn_lorder = 10;
+        int fsmn_dims = 512;
+        float cif_threshold = 1.0;
+        float tail_alphas = 0.45;
+        int asr_sample_rate = MODEL_SAMPLE_RATE;
+        int batch_size_ = 1;
+    };
+
+} // namespace funasr

runtime/onnxruntime/src/paraformer.cpp

@@ -0,0 +1,680 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#include "precomp.h"
+#include "paraformer.h"
+#include "encode_converter.h"
+#include <cstddef>
+
+using namespace std;
+namespace funasr {
+
+Paraformer::Paraformer()
+:use_hotword(false),
+ env_(ORT_LOGGING_LEVEL_ERROR, "paraformer"),session_options_{},
+ hw_env_(ORT_LOGGING_LEVEL_ERROR, "paraformer_hw"),hw_session_options{} {
+}
+
+// offline
+void Paraformer::InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){
+    LoadConfigFromYaml(am_config.c_str());
+    // knf options
+    fbank_opts_.frame_opts.dither = 0;
+    fbank_opts_.mel_opts.num_bins = n_mels;
+    fbank_opts_.frame_opts.samp_freq = asr_sample_rate;
+    fbank_opts_.frame_opts.window_type = window_type;
+    fbank_opts_.frame_opts.frame_shift_ms = frame_shift;
+    fbank_opts_.frame_opts.frame_length_ms = frame_length;
+    fbank_opts_.energy_floor = 0;
+    fbank_opts_.mel_opts.debug_mel = false;
+    // fbank_ = std::make_unique<knf::OnlineFbank>(fbank_opts);
+
+    // session_options_.SetInterOpNumThreads(1);
+    session_options_.SetIntraOpNumThreads(thread_num);
+    session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
+    // DisableCpuMemArena can improve performance
+    session_options_.DisableCpuMemArena();
+
+    try {
+        m_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(am_model).c_str(), session_options_);
+        LOG(INFO) << "Successfully load model from " << am_model;
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load am onnx model: " << e.what();
+        exit(-1);
+    }
+
+    GetInputNames(m_session_.get(), m_strInputNames, m_szInputNames);
+    GetOutputNames(m_session_.get(), m_strOutputNames, m_szOutputNames);
+    vocab = new Vocab(token_file.c_str());
+	phone_set_ = new PhoneSet(token_file.c_str());
+    LoadCmvn(am_cmvn.c_str());
+}
+
+// online
+void Paraformer::InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){
+    
+    LoadOnlineConfigFromYaml(am_config.c_str());
+    // knf options
+    fbank_opts_.frame_opts.dither = 0;
+    fbank_opts_.mel_opts.num_bins = n_mels;
+    fbank_opts_.frame_opts.samp_freq = asr_sample_rate;
+    fbank_opts_.frame_opts.window_type = window_type;
+    fbank_opts_.frame_opts.frame_shift_ms = frame_shift;
+    fbank_opts_.frame_opts.frame_length_ms = frame_length;
+    fbank_opts_.energy_floor = 0;
+    fbank_opts_.mel_opts.debug_mel = false;
+
+    // session_options_.SetInterOpNumThreads(1);
+    session_options_.SetIntraOpNumThreads(thread_num);
+    session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
+    // DisableCpuMemArena can improve performance
+    session_options_.DisableCpuMemArena();
+
+    try {
+        encoder_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(en_model).c_str(), session_options_);
+        LOG(INFO) << "Successfully load model from " << en_model;
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load am encoder model: " << e.what();
+        exit(-1);
+    }
+
+    try {
+        decoder_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(de_model).c_str(), session_options_);
+        LOG(INFO) << "Successfully load model from " << de_model;
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load am decoder model: " << e.what();
+        exit(-1);
+    }
+
+    // encoder
+    string strName;
+    GetInputName(encoder_session_.get(), strName);
+    en_strInputNames.push_back(strName.c_str());
+    GetInputName(encoder_session_.get(), strName,1);
+    en_strInputNames.push_back(strName);
+    
+    GetOutputName(encoder_session_.get(), strName);
+    en_strOutputNames.push_back(strName);
+    GetOutputName(encoder_session_.get(), strName,1);
+    en_strOutputNames.push_back(strName);
+    GetOutputName(encoder_session_.get(), strName,2);
+    en_strOutputNames.push_back(strName);
+
+    for (auto& item : en_strInputNames)
+        en_szInputNames_.push_back(item.c_str());
+    for (auto& item : en_strOutputNames)
+        en_szOutputNames_.push_back(item.c_str());
+
+    // decoder
+    int de_input_len = 4 + fsmn_layers;
+    int de_out_len = 2 + fsmn_layers;
+    for(int i=0;i<de_input_len; i++){
+        GetInputName(decoder_session_.get(), strName, i);
+        de_strInputNames.push_back(strName.c_str());
+    }
+
+    for(int i=0;i<de_out_len; i++){
+        GetOutputName(decoder_session_.get(), strName,i);
+        de_strOutputNames.push_back(strName);
+    }
+
+    for (auto& item : de_strInputNames)
+        de_szInputNames_.push_back(item.c_str());
+    for (auto& item : de_strOutputNames)
+        de_szOutputNames_.push_back(item.c_str());
+
+    vocab = new Vocab(token_file.c_str());
+    phone_set_ = new PhoneSet(token_file.c_str());
+    LoadCmvn(am_cmvn.c_str());
+}
+
+// 2pass
+void Paraformer::InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, 
+    const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, const std::string &online_token_file, int thread_num){
+    // online
+    InitAsr(en_model, de_model, am_cmvn, am_config, online_token_file, thread_num);
+
+    // offline
+    try {
+        m_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(am_model).c_str(), session_options_);
+        LOG(INFO) << "Successfully load model from " << am_model;
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load am onnx model: " << e.what();
+        exit(-1);
+    }
+
+    GetInputNames(m_session_.get(), m_strInputNames, m_szInputNames);
+    GetOutputNames(m_session_.get(), m_strOutputNames, m_szOutputNames);
+}
+
+void Paraformer::InitLm(const std::string &lm_file, 
+                        const std::string &lm_cfg_file, 
+                        const std::string &lex_file) {
+    try {
+        lm_ = std::shared_ptr<fst::Fst<fst::StdArc>>(
+            fst::Fst<fst::StdArc>::Read(lm_file));
+        if (lm_){
+            lm_vocab = new Vocab(lm_cfg_file.c_str(), lex_file.c_str());
+            LOG(INFO) << "Successfully load lm file " << lm_file;
+        }else{
+            LOG(ERROR) << "Failed to load lm file " << lm_file;
+        }
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load lm file: " << e.what();
+        exit(0);
+    }
+}
+
+void Paraformer::LoadConfigFromYaml(const char* filename){
+
+    YAML::Node config;
+    try{
+        config = YAML::LoadFile(filename);
+    }catch(exception const &e){
+        LOG(ERROR) << "Error loading file, yaml file error or not exist.";
+        exit(-1);
+    }
+
+    try{
+        YAML::Node frontend_conf = config["frontend_conf"];
+        this->asr_sample_rate = frontend_conf["fs"].as<int>();
+
+        YAML::Node lang_conf = config["lang"];
+        if (lang_conf.IsDefined()){
+            language = lang_conf.as<string>();
+        }
+    }catch(exception const &e){
+        LOG(ERROR) << "Error when load argument from vad config YAML.";
+        exit(-1);
+    }
+}
+
+void Paraformer::LoadOnlineConfigFromYaml(const char* filename){
+
+    YAML::Node config;
+    try{
+        config = YAML::LoadFile(filename);
+    }catch(exception const &e){
+        LOG(ERROR) << "Error loading file, yaml file error or not exist.";
+        exit(-1);
+    }
+
+    try{
+        YAML::Node frontend_conf = config["frontend_conf"];
+        YAML::Node encoder_conf = config["encoder_conf"];
+        YAML::Node decoder_conf = config["decoder_conf"];
+        YAML::Node predictor_conf = config["predictor_conf"];
+
+        this->window_type = frontend_conf["window"].as<string>();
+        this->n_mels = frontend_conf["n_mels"].as<int>();
+        this->frame_length = frontend_conf["frame_length"].as<int>();
+        this->frame_shift = frontend_conf["frame_shift"].as<int>();
+        this->lfr_m = frontend_conf["lfr_m"].as<int>();
+        this->lfr_n = frontend_conf["lfr_n"].as<int>();
+
+        this->encoder_size = encoder_conf["output_size"].as<int>();
+        this->fsmn_dims = encoder_conf["output_size"].as<int>();
+
+        this->fsmn_layers = decoder_conf["num_blocks"].as<int>();
+        this->fsmn_lorder = decoder_conf["kernel_size"].as<int>()-1;
+
+        this->cif_threshold = predictor_conf["threshold"].as<double>();
+        this->tail_alphas = predictor_conf["tail_threshold"].as<double>();
+
+        this->asr_sample_rate = frontend_conf["fs"].as<int>();
+
+
+    }catch(exception const &e){
+        LOG(ERROR) << "Error when load argument from vad config YAML.";
+        exit(-1);
+    }
+}
+
+void Paraformer::InitHwCompiler(const std::string &hw_model, int thread_num) {
+    hw_session_options.SetIntraOpNumThreads(thread_num);
+    hw_session_options.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
+    // DisableCpuMemArena can improve performance
+    hw_session_options.DisableCpuMemArena();
+
+    try {
+        hw_m_session = std::make_unique<Ort::Session>(hw_env_, ORTSTRING(hw_model).c_str(), hw_session_options);
+        LOG(INFO) << "Successfully load model from " << hw_model;
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load hw compiler onnx model: " << e.what();
+        exit(-1);
+    }
+
+    string strName;
+    GetInputName(hw_m_session.get(), strName);
+    hw_m_strInputNames.push_back(strName.c_str());
+    //GetInputName(hw_m_session.get(), strName,1);
+    //hw_m_strInputNames.push_back(strName);
+    
+    GetOutputName(hw_m_session.get(), strName);
+    hw_m_strOutputNames.push_back(strName);
+
+    for (auto& item : hw_m_strInputNames)
+        hw_m_szInputNames.push_back(item.c_str());
+    for (auto& item : hw_m_strOutputNames)
+        hw_m_szOutputNames.push_back(item.c_str());
+    // if init hotword compiler is called, this is a hotword paraformer model
+    use_hotword = true;
+}
+
+void Paraformer::InitSegDict(const std::string &seg_dict_model) {
+    seg_dict = new SegDict(seg_dict_model.c_str());
+}
+
+Paraformer::~Paraformer()
+{
+    if(vocab){
+        delete vocab;
+    }
+    if(lm_vocab){
+        delete lm_vocab;
+    }
+    if(seg_dict){
+        delete seg_dict;
+    }
+    if(phone_set_){
+        delete phone_set_;
+    }
+}
+
+void Paraformer::StartUtterance()
+{
+}
+
+void Paraformer::EndUtterance()
+{
+}
+
+void Paraformer::Reset()
+{
+}
+
+void Paraformer::FbankKaldi(float sample_rate, const float* waves, int len, std::vector<std::vector<float>> &asr_feats) {
+    knf::OnlineFbank fbank_(fbank_opts_);
+    std::vector<float> buf(len);
+    for (int32_t i = 0; i != len; ++i) {
+        buf[i] = waves[i] * 32768;
+    }
+    fbank_.AcceptWaveform(sample_rate, buf.data(), buf.size());
+
+    int32_t frames = fbank_.NumFramesReady();
+    for (int32_t i = 0; i != frames; ++i) {
+        const float *frame = fbank_.GetFrame(i);
+        std::vector<float> frame_vector(frame, frame + fbank_opts_.mel_opts.num_bins);
+        asr_feats.emplace_back(frame_vector);
+    }
+}
+
+void Paraformer::LoadCmvn(const char *filename)
+{
+    ifstream cmvn_stream(filename);
+    if (!cmvn_stream.is_open()) {
+        LOG(ERROR) << "Failed to open file: " << filename;
+        exit(-1);
+    }
+    string line;
+
+    while (getline(cmvn_stream, line)) {
+        istringstream iss(line);
+        vector<string> line_item{istream_iterator<string>{iss}, istream_iterator<string>{}};
+        if (line_item[0] == "<AddShift>") {
+            getline(cmvn_stream, line);
+            istringstream means_lines_stream(line);
+            vector<string> means_lines{istream_iterator<string>{means_lines_stream}, istream_iterator<string>{}};
+            if (means_lines[0] == "<LearnRateCoef>") {
+                for (int j = 3; j < means_lines.size() - 1; j++) {
+                    means_list_.push_back(stof(means_lines[j]));
+                }
+                continue;
+            }
+        }
+        else if (line_item[0] == "<Rescale>") {
+            getline(cmvn_stream, line);
+            istringstream vars_lines_stream(line);
+            vector<string> vars_lines{istream_iterator<string>{vars_lines_stream}, istream_iterator<string>{}};
+            if (vars_lines[0] == "<LearnRateCoef>") {
+                for (int j = 3; j < vars_lines.size() - 1; j++) {
+                    vars_list_.push_back(stof(vars_lines[j])*scale);
+                }
+                continue;
+            }
+        }
+    }
+}
+
+string Paraformer::GreedySearch(float * in, int n_len,  int64_t token_nums, bool is_stamp, std::vector<float> us_alphas, std::vector<float> us_cif_peak)
+{
+    vector<int> hyps;
+    int Tmax = n_len;
+    for (int i = 0; i < Tmax; i++) {
+        int max_idx;
+        float max_val;
+        FindMax(in + i * token_nums, token_nums, max_val, max_idx);
+        hyps.push_back(max_idx);
+    }
+    if(!is_stamp){
+        return vocab->Vector2StringV2(hyps, language);
+    }else{
+        std::vector<string> char_list;
+        std::vector<std::vector<float>> timestamp_list;
+        std::string res_str;
+        vocab->Vector2String(hyps, char_list);
+        std::vector<string> raw_char(char_list);
+        TimestampOnnx(us_alphas, us_cif_peak, char_list, res_str, timestamp_list);
+
+        return PostProcess(raw_char, timestamp_list);
+    }
+}
+
+string Paraformer::BeamSearch(WfstDecoder* &wfst_decoder, float *in, int len, int64_t token_nums)
+{
+  return wfst_decoder->Search(in, len, token_nums);
+}
+
+string Paraformer::FinalizeDecode(WfstDecoder* &wfst_decoder,
+                                  bool is_stamp, std::vector<float> us_alphas, std::vector<float> us_cif_peak)
+{
+  return wfst_decoder->FinalizeDecode(is_stamp, us_alphas, us_cif_peak);
+}
+
+void Paraformer::LfrCmvn(std::vector<std::vector<float>> &asr_feats) {
+
+    std::vector<std::vector<float>> out_feats;
+    int T = asr_feats.size();
+    int T_lrf = ceil(1.0 * T / lfr_n);
+
+    // Pad frames at start(copy first frame)
+    for (int i = 0; i < (lfr_m - 1) / 2; i++) {
+        asr_feats.insert(asr_feats.begin(), asr_feats[0]);
+    }
+    // Merge lfr_m frames as one,lfr_n frames per window
+    T = T + (lfr_m - 1) / 2;
+    std::vector<float> p;
+    for (int i = 0; i < T_lrf; i++) {
+        if (lfr_m <= T - i * lfr_n) {
+            for (int j = 0; j < lfr_m; j++) {
+                p.insert(p.end(), asr_feats[i * lfr_n + j].begin(), asr_feats[i * lfr_n + j].end());
+            }
+            out_feats.emplace_back(p);
+            p.clear();
+        } else {
+            // Fill to lfr_m frames at last window if less than lfr_m frames  (copy last frame)
+            int num_padding = lfr_m - (T - i * lfr_n);
+            for (int j = 0; j < (asr_feats.size() - i * lfr_n); j++) {
+                p.insert(p.end(), asr_feats[i * lfr_n + j].begin(), asr_feats[i * lfr_n + j].end());
+            }
+            for (int j = 0; j < num_padding; j++) {
+                p.insert(p.end(), asr_feats[asr_feats.size() - 1].begin(), asr_feats[asr_feats.size() - 1].end());
+            }
+            out_feats.emplace_back(p);
+            p.clear();
+        }
+    }
+    // Apply cmvn
+    for (auto &out_feat: out_feats) {
+        for (int j = 0; j < means_list_.size(); j++) {
+            out_feat[j] = (out_feat[j] + means_list_[j]) * vars_list_[j];
+        }
+    }
+    asr_feats = out_feats;
+}
+
+std::vector<std::string> Paraformer::Forward(float** din, int* len, bool input_finished, const std::vector<std::vector<float>> &hw_emb, void* decoder_handle, int batch_in)
+{
+    std::vector<std::string> results;
+    string result="";
+    WfstDecoder* wfst_decoder = (WfstDecoder*)decoder_handle;
+    int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins;
+
+    if(batch_in != 1){
+        results.push_back(result);
+        return results;
+    }
+
+    std::vector<std::vector<float>> asr_feats;
+    FbankKaldi(asr_sample_rate, din[0], len[0], asr_feats);
+    if(asr_feats.size() == 0){
+        results.push_back(result);
+        return results;
+    }
+    LfrCmvn(asr_feats);
+    int32_t feat_dim = lfr_m*in_feat_dim;
+    int32_t num_frames = asr_feats.size();
+
+    std::vector<float> wav_feats;
+    for (const auto &frame_feat: asr_feats) {
+        wav_feats.insert(wav_feats.end(), frame_feat.begin(), frame_feat.end());
+    }
+
+#ifdef _WIN_X86
+        Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
+#else
+        Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
+#endif
+
+    const int64_t input_shape_[3] = {1, num_frames, feat_dim};
+    Ort::Value onnx_feats = Ort::Value::CreateTensor<float>(m_memoryInfo,
+        wav_feats.data(),
+        wav_feats.size(),
+        input_shape_,
+        3);
+
+    const int64_t paraformer_length_shape[1] = {1};
+    std::vector<int32_t> paraformer_length;
+    paraformer_length.emplace_back(num_frames);
+    Ort::Value onnx_feats_len = Ort::Value::CreateTensor<int32_t>(
+          m_memoryInfo, paraformer_length.data(), paraformer_length.size(), paraformer_length_shape, 1);
+
+    std::vector<Ort::Value> input_onnx;
+    input_onnx.emplace_back(std::move(onnx_feats));
+    input_onnx.emplace_back(std::move(onnx_feats_len));
+
+    std::vector<float> embedding;
+    try{
+        if (use_hotword) {
+            if(hw_emb.size()<=0){
+                LOG(ERROR) << "hw_emb is null";
+                results.push_back(result);
+                return results;
+            }
+            //PrintMat(hw_emb, "input_clas_emb");
+            const int64_t hotword_shape[3] = {1, static_cast<int64_t>(hw_emb.size()), static_cast<int64_t>(hw_emb[0].size())};
+            embedding.reserve(hw_emb.size() * hw_emb[0].size());
+            for (auto item : hw_emb) {
+                embedding.insert(embedding.end(), item.begin(), item.end());
+            }
+            //LOG(INFO) << "hotword shape " << hotword_shape[0] << " " << hotword_shape[1] << " " << hotword_shape[2] << " size " << embedding.size();
+            Ort::Value onnx_hw_emb = Ort::Value::CreateTensor<float>(
+                m_memoryInfo, embedding.data(), embedding.size(), hotword_shape, 3);
+
+            input_onnx.emplace_back(std::move(onnx_hw_emb));
+        }
+    }catch (std::exception const &e)
+    {
+        LOG(ERROR)<<e.what();
+        results.push_back(result);
+        return results;
+    }
+
+    try {
+        auto outputTensor = m_session_->Run(Ort::RunOptions{nullptr}, m_szInputNames.data(), input_onnx.data(), input_onnx.size(), m_szOutputNames.data(), m_szOutputNames.size());
+        std::vector<int64_t> outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape();
+        //LOG(INFO) << "paraformer out shape " << outputShape[0] << " " << outputShape[1] << " " << outputShape[2];
+
+        int64_t outputCount = std::accumulate(outputShape.begin(), outputShape.end(), 1, std::multiplies<int64_t>());
+        float* floatData = outputTensor[0].GetTensorMutableData<float>();
+        auto encoder_out_lens = outputTensor[1].GetTensorMutableData<int64_t>();
+        // timestamp
+        if(outputTensor.size() == 4){
+            std::vector<int64_t> us_alphas_shape = outputTensor[2].GetTensorTypeAndShapeInfo().GetShape();
+            float* us_alphas_data = outputTensor[2].GetTensorMutableData<float>();
+            std::vector<float> us_alphas(us_alphas_shape[1]);
+            for (int i = 0; i < us_alphas_shape[1]; i++) {
+                us_alphas[i] = us_alphas_data[i];
+            }
+
+            std::vector<int64_t> us_peaks_shape = outputTensor[3].GetTensorTypeAndShapeInfo().GetShape();
+            float* us_peaks_data = outputTensor[3].GetTensorMutableData<float>();
+            std::vector<float> us_peaks(us_peaks_shape[1]);
+            for (int i = 0; i < us_peaks_shape[1]; i++) {
+                us_peaks[i] = us_peaks_data[i];
+            }
+			if (lm_ == nullptr) {
+                result = GreedySearch(floatData, *encoder_out_lens, outputShape[2], true, us_alphas, us_peaks);
+			} else {
+			    result = BeamSearch(wfst_decoder, floatData, *encoder_out_lens, outputShape[2]);
+                if (input_finished) {
+                    result = FinalizeDecode(wfst_decoder, true, us_alphas, us_peaks);
+                }
+			}
+        }else{
+			if (lm_ == nullptr) {
+                result = GreedySearch(floatData, *encoder_out_lens, outputShape[2]);
+			} else {
+			    result = BeamSearch(wfst_decoder, floatData, *encoder_out_lens, outputShape[2]);
+                if (input_finished) {
+                    result = FinalizeDecode(wfst_decoder);
+                }
+			}
+        }
+    }
+    catch (std::exception const &e)
+    {
+        LOG(ERROR)<<e.what();
+    }
+
+    results.push_back(result);
+    return results;
+}
+
+
+std::vector<std::vector<float>> Paraformer::CompileHotwordEmbedding(std::string &hotwords) {
+    int embedding_dim = encoder_size;
+    std::vector<std::vector<float>> hw_emb;
+    if (!use_hotword) {
+        std::vector<float> vec(embedding_dim, 0);
+        hw_emb.push_back(vec);
+        return hw_emb;
+    }
+    int max_hotword_len = 10;
+    std::vector<int32_t> hotword_matrix;
+    std::vector<int32_t> lengths;
+    int hotword_size = 1;
+    int real_hw_size = 0;
+    if (!hotwords.empty()) {
+      std::vector<std::string> hotword_array = split(hotwords, ' ');
+      hotword_size = hotword_array.size() + 1;
+      hotword_matrix.reserve(hotword_size * max_hotword_len);
+      for (auto hotword : hotword_array) {
+        std::vector<std::string> chars;
+        if (EncodeConverter::IsAllChineseCharactor((const U8CHAR_T*)hotword.c_str(), hotword.size())) {
+          KeepChineseCharacterAndSplit(hotword, chars);
+        } else {
+          // for english
+          std::vector<std::string> words = split(hotword, ' ');
+          for (auto word : words) {
+            std::vector<string> tokens = seg_dict->GetTokensByWord(word);
+            chars.insert(chars.end(), tokens.begin(), tokens.end());
+          }
+        }
+        if(chars.size()==0){
+            continue;
+        }
+        std::vector<int32_t> hw_vector(max_hotword_len, 0);
+        int vector_len = std::min(max_hotword_len, (int)chars.size());
+        int chs_oov = false;
+        for (int i=0; i<vector_len; i++) {
+          hw_vector[i] = phone_set_->String2Id(chars[i]);
+          if(hw_vector[i] == -1){
+            chs_oov = true;
+            break;
+          }
+        }
+        if(chs_oov){
+          LOG(INFO) << "OOV: " << hotword;
+          continue;
+        }
+        LOG(INFO) << hotword;
+        lengths.push_back(vector_len);
+        real_hw_size += 1;
+        hotword_matrix.insert(hotword_matrix.end(), hw_vector.begin(), hw_vector.end());
+      }
+      hotword_size = real_hw_size + 1;
+    }
+    std::vector<int32_t> blank_vec(max_hotword_len, 0);
+    blank_vec[0] = 1;
+    hotword_matrix.insert(hotword_matrix.end(), blank_vec.begin(), blank_vec.end());
+    lengths.push_back(1);
+
+#ifdef _WIN_X86
+        Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
+#else
+        Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
+#endif
+
+    const int64_t input_shape_[2] = {hotword_size, max_hotword_len};
+    Ort::Value onnx_hotword = Ort::Value::CreateTensor<int32_t>(m_memoryInfo,
+        (int32_t*)hotword_matrix.data(),
+        hotword_size * max_hotword_len,
+        input_shape_,
+        2);
+    LOG(INFO) << "clas shape " << hotword_size << " " << max_hotword_len << std::endl;
+    
+    std::vector<Ort::Value> input_onnx;
+    input_onnx.emplace_back(std::move(onnx_hotword));
+
+    std::vector<std::vector<float>> result;
+    try {
+        auto outputTensor = hw_m_session->Run(Ort::RunOptions{nullptr}, hw_m_szInputNames.data(), input_onnx.data(), input_onnx.size(), hw_m_szOutputNames.data(), hw_m_szOutputNames.size());
+        std::vector<int64_t> outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape();
+
+        int64_t outputCount = std::accumulate(outputShape.begin(), outputShape.end(), 1, std::multiplies<int64_t>());
+        float* floatData = outputTensor[0].GetTensorMutableData<float>(); // shape [max_hotword_len, hotword_size, dim]
+        // get embedding by real hotword length
+        assert(outputShape[0] == max_hotword_len);
+        assert(outputShape[1] == hotword_size);
+        embedding_dim = outputShape[2];
+
+        for (int j = 0; j < hotword_size; j++)
+        {
+            int start_pos = hotword_size * (lengths[j] - 1) * embedding_dim + j * embedding_dim;
+            std::vector<float> embedding;
+            embedding.insert(embedding.begin(), floatData + start_pos, floatData + start_pos + embedding_dim);
+            result.push_back(embedding);
+        }
+    }
+    catch (std::exception const &e)
+    {
+        LOG(ERROR)<<e.what();
+    }
+    //PrintMat(result, "clas_embedding_output");
+    return result;
+}
+
+Vocab* Paraformer::GetVocab()
+{
+    return vocab;
+}
+
+Vocab* Paraformer::GetLmVocab()
+{
+    return lm_vocab;
+}
+
+PhoneSet* Paraformer::GetPhoneSet()
+{
+    return phone_set_;
+}
+
+string Paraformer::Rescoring()
+{
+    LOG(ERROR)<<"Not Imp!!!!!!";
+    return "";
+}
+} // namespace funasr

runtime/onnxruntime/src/paraformer.h

@@ -0,0 +1,118 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+#pragma once
+
+#include "precomp.h"
+#include "fst/fstlib.h"
+#include "fst/symbol-table.h"
+#include "bias-lm.h"
+#include "phone-set.h"
+
+namespace funasr {
+
+    class Paraformer : public Model {
+    /**
+     * Author: Speech Lab of DAMO Academy, Alibaba Group
+     * Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition
+     * https://arxiv.org/pdf/2206.08317.pdf
+    */
+    private:
+        Vocab* vocab = nullptr;
+        Vocab* lm_vocab = nullptr;
+        SegDict* seg_dict = nullptr;
+        PhoneSet* phone_set_ = nullptr;
+        //const float scale = 22.6274169979695;
+        const float scale = 1.0;
+
+        void LoadConfigFromYaml(const char* filename);
+        void LoadOnlineConfigFromYaml(const char* filename);
+        void LoadCmvn(const char *filename);
+        void LfrCmvn(std::vector<std::vector<float>> &asr_feats);
+
+        std::shared_ptr<Ort::Session> hw_m_session = nullptr;
+        Ort::Env hw_env_;
+        Ort::SessionOptions hw_session_options;
+        vector<string> hw_m_strInputNames, hw_m_strOutputNames;
+        vector<const char*> hw_m_szInputNames;
+        vector<const char*> hw_m_szOutputNames;
+        bool use_hotword;
+
+    public:
+        Paraformer();
+        ~Paraformer();
+        void InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);
+        // online
+        void InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);
+        // 2pass
+        void InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, 
+            const std::string &am_config, const std::string &token_file, const std::string &online_token_file, int thread_num);
+        void InitHwCompiler(const std::string &hw_model, int thread_num);
+        void InitSegDict(const std::string &seg_dict_model);
+        std::vector<std::vector<float>> CompileHotwordEmbedding(std::string &hotwords);
+        void Reset();
+        void FbankKaldi(float sample_rate, const float* waves, int len, std::vector<std::vector<float>> &asr_feats);
+        std::vector<std::string> Forward(float** din, int* len, bool input_finished=true, const std::vector<std::vector<float>> &hw_emb={{0.0}}, void* wfst_decoder=nullptr, int batch_in=1);
+        string GreedySearch( float* in, int n_len, int64_t token_nums,
+                             bool is_stamp=false, std::vector<float> us_alphas={0}, std::vector<float> us_cif_peak={0});
+
+        string Rescoring();
+        string GetLang(){return language;};
+        int GetAsrSampleRate() { return asr_sample_rate; };
+        int GetBatchSize() {return batch_size_;};
+        void StartUtterance();
+        void EndUtterance();
+        void InitLm(const std::string &lm_file, const std::string &lm_cfg_file, const std::string &lex_file);
+        string BeamSearch(WfstDecoder* &wfst_decoder, float* in, int n_len, int64_t token_nums);
+        string FinalizeDecode(WfstDecoder* &wfst_decoder,
+                          bool is_stamp=false, std::vector<float> us_alphas={0}, std::vector<float> us_cif_peak={0});
+        Vocab* GetVocab();
+        Vocab* GetLmVocab();
+        PhoneSet* GetPhoneSet();
+		
+        knf::FbankOptions fbank_opts_;
+        vector<float> means_list_;
+        vector<float> vars_list_;
+        int lfr_m = PARA_LFR_M;
+        int lfr_n = PARA_LFR_N;
+
+        // paraformer-offline
+        std::shared_ptr<Ort::Session> m_session_ = nullptr;
+        Ort::Env env_;
+        Ort::SessionOptions session_options_;
+
+        vector<string> m_strInputNames, m_strOutputNames;
+        vector<const char*> m_szInputNames;
+        vector<const char*> m_szOutputNames;
+
+        std::string language="zh-cn";
+
+        // paraformer-online
+        std::shared_ptr<Ort::Session> encoder_session_ = nullptr;
+        std::shared_ptr<Ort::Session> decoder_session_ = nullptr;
+        vector<string> en_strInputNames, en_strOutputNames;
+        vector<const char*> en_szInputNames_;
+        vector<const char*> en_szOutputNames_;
+        vector<string> de_strInputNames, de_strOutputNames;
+        vector<const char*> de_szInputNames_;
+        vector<const char*> de_szOutputNames_;
+
+        // lm
+        std::shared_ptr<fst::Fst<fst::StdArc>> lm_ = nullptr;
+
+        string window_type = "hamming";
+        int frame_length = 25;
+        int frame_shift = 10;
+        int n_mels = 80;
+        int encoder_size = 512;
+        int fsmn_layers = 16;
+        int fsmn_lorder = 10;
+        int fsmn_dims = 512;
+        float cif_threshold = 1.0;
+        float tail_alphas = 0.45;
+        int asr_sample_rate = MODEL_SAMPLE_RATE;
+        int batch_size_ = 1;
+    };
+
+} // namespace funasr

runtime/onnxruntime/src/phone-set.cpp

@@ -0,0 +1,95 @@
+#include "phone-set.h"
+#include <yaml-cpp/yaml.h>
+#include <glog/logging.h>
+
+#include <fstream>
+#include <iostream>
+#include <list>
+#include <sstream>
+#include <string>
+
+using namespace std;
+
+namespace funasr {
+PhoneSet::PhoneSet(const char *filename) {
+  ifstream in(filename);
+  LoadPhoneSetFromJson(filename);
+}
+PhoneSet::~PhoneSet()
+{
+}
+
+void PhoneSet::LoadPhoneSetFromYaml(const char* filename) {
+  YAML::Node config;
+  try{
+    config = YAML::LoadFile(filename);
+  }catch(exception const &e){
+     LOG(INFO) << "Error loading file, yaml file error or not exist.";
+     exit(-1);
+  }
+  YAML::Node myList = config["token_list"];
+  int id = 0;
+  for (YAML::const_iterator it = myList.begin(); it != myList.end(); ++it, id++) {
+    phone_.push_back(it->as<string>());
+    phn2Id_.emplace(it->as<string>(), id);
+  }
+}
+
+void PhoneSet::LoadPhoneSetFromJson(const char* filename) {
+    nlohmann::json json_array;
+    std::ifstream file(filename);
+    if (file.is_open()) {
+        file >> json_array;
+        file.close();
+    } else {
+        LOG(INFO) << "Error loading token file, token file error or not exist.";
+        exit(-1);
+    }
+
+    int id = 0;
+    for (const auto& element : json_array) {
+        phone_.push_back(element);
+        phn2Id_.emplace(element, id);
+        id++;
+    }
+}
+
+int PhoneSet::Size() const {
+  return phone_.size();
+}
+
+int PhoneSet::String2Id(string phn_str) const {
+  if (phn2Id_.count(phn_str)) {
+    return phn2Id_.at(phn_str);
+  } else {
+    //LOG(INFO) << "Phone unit not exist.";
+    return -1;
+  }
+}
+
+string PhoneSet::Id2String(int id) const {
+  if (id < 0 || id > Size()) {
+    //LOG(INFO) << "Phone id not exist.";
+    return "";
+  } else {
+    return phone_[id];
+  }
+}
+
+bool PhoneSet::Find(string phn_str) const {
+  return phn2Id_.count(phn_str) > 0;
+}
+
+int PhoneSet::GetBegSilPhnId() const {
+  return String2Id(UNIT_BEG_SIL_SYMBOL);
+}
+
+int PhoneSet::GetEndSilPhnId() const {
+  return String2Id(UNIT_END_SIL_SYMBOL);
+}
+
+int PhoneSet::GetBlkPhnId() const {
+  return String2Id(UNIT_BLK_SYMBOL);
+}
+
+}

runtime/onnxruntime/src/phone-set.h

@@ -0,0 +1,36 @@
+#ifndef PHONESET_H
+#define PHONESET_H
+
+#include <stdint.h>
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include "nlohmann/json.hpp"
+#define UNIT_BEG_SIL_SYMBOL "<s>"
+#define UNIT_END_SIL_SYMBOL "</s>"
+#define UNIT_BLK_SYMBOL "<blank>"
+
+using namespace std;
+
+namespace funasr {
+class PhoneSet {
+  public:
+    PhoneSet(const char *filename);
+    ~PhoneSet();
+    int Size() const;
+    int String2Id(string str) const;
+    string Id2String(int id) const;
+    bool Find(string str) const;
+    int GetBegSilPhnId() const;
+    int GetEndSilPhnId() const;
+    int GetBlkPhnId() const;
+
+  private:
+    vector<string> phone_;
+    unordered_map<string, int> phn2Id_;
+    void LoadPhoneSetFromYaml(const char* filename);
+    void LoadPhoneSetFromJson(const char* filename);
+};
+
+} // namespace funasr
+#endif

runtime/onnxruntime/src/precomp.h

@@ -0,0 +1,75 @@
+#pragma once 
+// system 
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <deque>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <iterator>
+#include <list>
+#include <locale.h>
+#include <vector>
+#include <string>
+#include <math.h>
+#include <numeric>
+#include <cstring>
+
+#ifdef _WIN32
+#include <win_func.h>
+#else
+#include <unistd.h>
+#endif
+
+using namespace std;
+// third part
+#if defined(__APPLE__)
+#include <onnxruntime/onnxruntime_cxx_api.h>
+#else
+#include "onnxruntime_run_options_config_keys.h"
+#include "onnxruntime_cxx_api.h"
+#include "itn-model.h"
+#include "itn-processor.h"
+#endif
+
+#include "kaldi-native-fbank/csrc/feature-fbank.h"
+#include "kaldi-native-fbank/csrc/online-feature.h"
+#include "kaldi/decoder/lattice-faster-online-decoder.h"
+// mine
+#include <glog/logging.h>
+
+
+#include "common-struct.h"
+#include "com-define.h"
+#include "commonfunc.h"
+#include "predefine-coe.h"
+#include "model.h"
+#include "vad-model.h"
+#include "punc-model.h"
+#include "tokenizer.h"
+#include "ct-transformer.h"
+#include "ct-transformer-online.h"
+#include "e2e-vad.h"
+#include "fsmn-vad.h"
+#include "encode_converter.h"
+#include "vocab.h"
+#include "phone-set.h"
+#include "wfst-decoder.h"
+#include "audio.h"
+#include "fsmn-vad-online.h"
+#include "tensor.h"
+#include "util.h"
+#include "seg_dict.h"
+#include "resample.h"
+#include "paraformer.h"
+#include "sensevoice-small.h"
+#ifdef USE_GPU
+#include "paraformer-torch.h"
+#endif
+#include "paraformer-online.h"
+#include "offline-stream.h"
+#include "tpass-stream.h"
+#include "tpass-online-stream.h"
+#include "funasrruntime.h"

runtime/onnxruntime/src/predefine-coe.h

@@ -0,0 +1,595 @@
+#ifndef PREDEFINE_COE_H
+#define PREDEFINE_COE_H
+
+#include <stdint.h>
+
+namespace funasr {
+const int32_t melcoe_hex[] = {
+
+    0x3f01050c, 0x3e0afb11, 0x3f5d413c, 0x3f547fd0, 0x3e2e00c1, 0x3f132970,
+    0x3ed9ad21, 0x3ebb8bb9, 0x3f223a24, 0x3e4de6f8, 0x3f4c8642, 0x3d9c0424,
+    0x3f6c7f7c, 0x3f7d295a, 0x3c35a961, 0x3f6fd497, 0x3d815b45, 0x3f6af197,
+    0x3da87344, 0x3f6dfce9, 0x3d9018b9, 0x3f787ebc, 0x3d2098fe, 0x3cf02873,
+    0x3f75f670, 0x3e08e423, 0x3f5dc6f7, 0x3e8161eb, 0x3f3f4f0b, 0x3eca38e2,
+    0x3f1ae38f, 0x3f0f2d23, 0x3ee1a5ba, 0x3f3e9a98, 0x3e82cad1, 0x3f7321ac,
+    0x3e321028, 0x3d4de548, 0x3f537bf6, 0x3ed50f76, 0x3f157845, 0x3f2cf6bc,
+    0x3ea61288, 0x3f739ea7, 0x3e794186, 0x3d461590, 0x3f41af9f, 0x3f0cdfd4,
+    0x3ee64058, 0x3f5f23aa, 0x3e53d467, 0x3e037156, 0x3f4b0ae6, 0x3f0e2fac,
+    0x3ee3a0a8, 0x3f6ab111, 0x3e94b1ed, 0x3daa7774, 0x3f35a70a, 0x3f2d08dc,
+    0x3d951fb4, 0x3ea5ee48, 0x3f6d5c09, 0x3ef61e1a, 0x3f04f0f3, 0x3f66305c,
+    0x3ea7def9, 0x3dce7d20, 0x3f2c1083, 0x3f44354b, 0x3e5baf49, 0x3e6f2ad2,
+    0x3f49142e, 0x3f2bfe35, 0x3e0d627b, 0x3ea80396, 0x3f5ca761, 0x3f1ce830,
+    0x3dc4d786, 0x3ec62fa0, 0x3f67650f, 0x3f165fc0, 0x3db1323f, 0x3ed34080,
+    0x3f69d9b8, 0x3f17def1, 0x3ddbd6b6, 0x3ed0421e, 0x3f648529, 0x3f20ebbd,
+    0x3e20901a, 0x3ebe2886, 0x3f57dbf9, 0x3f3116ac, 0x3e6edcc6, 0x3e9dd2a9,
+    0x3f4448ce, 0x3f47f9a3, 0x3eaba511, 0x3e601974, 0x3f2a2d77, 0x3f6536e2,
+    0x3eec3842, 0x3d0781f6, 0x3dd648ed, 0x3f09e3df, 0x3f7787e1, 0x3f1c411f,
+    0x3e45b702, 0x3ec77dc2, 0x3f4e9240, 0x3f47f500, 0x3ebf9c61, 0x3e602c00,
+    0x3f2031d0, 0x3f78f0f7, 0x3f135547, 0x3e3bcd78, 0x3ce1e12a, 0x3ed95573,
+    0x3f510ca2, 0x3f4bc3c2, 0x3ed37e77, 0x3d0ded37, 0x3e50f0f8, 0x3f1640c5,
+    0x3f77212d, 0x3f291bd1, 0x3e94df6c, 0x3eadc85e, 0x3f35904a, 0x3f6cd43b,
+    0x3f104351, 0x3e52dc63, 0x3d995e26, 0x3edf795f, 0x3f4b48e7, 0x3f5a29e7,
+    0x3f00963d, 0x3e1fdb2f, 0x3e175865, 0x3efed385, 0x3f580934, 0x3f50466d,
+    0x3ef30046, 0x3e0e7c6b, 0x3e3ee64e, 0x3f067fdd, 0x3f5c60e5, 0x3f4e9ea4,
+    0x3ef4f46a, 0x3e1cb596, 0x3e45856f, 0x3f0585cb, 0x3f58d29b, 0x3f54b3ef,
+    0x3f0309ad, 0x3e48aa5b, 0x3e2d3042, 0x3ef9eca6, 0x3f4dd569, 0x3f6212c4,
+    0x3f12be68, 0x3e8853a3, 0x3def69e0, 0x3eda8330, 0x3f3bd62e, 0x3f76516a,
+    0x3f2931b5, 0x3eb98e9b, 0x3d88773c, 0x3d1ae95c, 0x3ead9c96, 0x3f2338b2,
+    0x3f6ef119, 0x3f46054d, 0x3ef74eba, 0x3e47c83a, 0x3e67eace, 0x3f0458a3,
+    0x3f4e0df1, 0x3f68e26b, 0x3f207590, 0x3eb1515d, 0x3d8bc852, 0x3db8eca9,
+    0x3ebf14e0, 0x3f275751, 0x3f6e86f6, 0x3f4ae3f8, 0x3f04e6de, 0x3e7dfcce,
+    0x3e547020, 0x3ef63244, 0x3f4080cd, 0x3f7aaa80, 0x3f366659, 0x3ee560cb,
+    0x3e3e1967, 0x3caab00e, 0x3e93334e, 0x3f0d4f9a, 0x3f5079a6, 0x3f6ce5f8,
+    0x3f2acd10, 0x3ed272ff, 0x3e20a4c5, 0x3d98d042, 0x3eaa65e0, 0x3f16c680,
+    0x3f57d6cf, 0x3f679a1b, 0x3f278a40, 0x3ecfef5c, 0x3e2381fd, 0x3dc32f28,
+    0x3eb0eb80, 0x3f180852, 0x3f571f81, 0x3f6a42d8, 0x3f2c1ce8, 0x3edcd9d1,
+    0x3e44c475, 0x3dade93f, 0x3ea7c630, 0x3f119318, 0x3f4ecee3, 0x3f7467d4,
+    0x3f380f62, 0x3ef84c54, 0x3e815525, 0x3cb361d7, 0x3d3982c8, 0x3e8fe13b,
+    0x3f03d9d6, 0x3f3f556d, 0x3f7a64f1, 0x3f4af618, 0x3f10ba30, 0x3eadcbc5,
+    0x3debbe02, 0x3e5427a0, 0x3ede8b9f, 0x3f291a1d, 0x3f628840, 0x3f646e63,
+    0x3f2bc86b, 0x3ee70902, 0x3e6e854e, 0x3c83b300, 0x3ddc8cea, 0x3ea86f2a,
+    0x3f0c7b7f, 0x3f445eac, 0x3f7be268, 0x3f4cf80b, 0x3f162f6e, 0x3ebf8516,
+    0x3e26c0c2, 0x3e4c1fd5, 0x3ed3a124, 0x3f203d75, 0x3f564fd0, 0x3f73f733,
+    0x3f3e966d, 0x3f098cbf, 0x3ea9b21c, 0x3e01e917, 0x3d408cd1, 0x3e82d326,
+    0x3eece682, 0x3f2b26f2, 0x3f5f85ba, 0x3f6c6f56, 0x3f38b733, 0x3f0550d9,
+    0x3ea47689, 0x3dfbabd7, 0x3d9c8552, 0x3e8e919a, 0x3ef55e4f, 0x3f2dc4bb,
+    0x3f608a85, 0x3f6cfe84, 0x3f3ad56c, 0x3f08f945, 0x3eaed247, 0x3e189086,
+    0x3d980be2, 0x3e8a5528, 0x3eee0d76, 0x3f2896dc, 0x3f59dbde, 0x3f75295d,
+    0x3f4477f6, 0x3f140f14, 0x3ec7dbbd, 0x3e504e0f, 0x3c8fe67e, 0x3d2d6a38,
+    0x3e6e2028, 0x3ed7e1d9, 0x3f1c1221, 0x3f4bec7c, 0x3f7b80cc, 0x3f553023,
+    0x3f262589, 0x3eeebd40, 0x3e91b54d, 0x3dd4c6f8, 0x3e2b3f74, 0x3eb3b4ef,
+    0x3f08a160, 0x3f372559, 0x3f656721, 0x3f6c988d, 0x3f3ed8f9, 0x3f11596d,
+    0x3ec83270, 0x3e5c5ea4, 0x3d254149, 0x3d9b3b97, 0x3e824e0e, 0x3edd4d25,
+    0x3f1be6c8, 0x3f48e857, 0x3f75abeb, 0x3f5dcdd1, 0x3f318436, 0x3f0576a0,
+    0x3eb348db, 0x3e3833fb, 0x3c2bedc9, 0x3e08c8be, 0x3e9cf794, 0x3ef512c0,
+    0x3f265b92, 0x3f51f301, 0x3f7d5049, 0x3f578bfc, 0x3f2ca136, 0x3f01eecf,
+    0x3eaee867, 0x3e34c34c, 0x3c490794, 0x3e21d00f, 0x3ea6bd94, 0x3efc2262,
+    0x3f288bcc, 0x3f52cf2d, 0x3f7cdbe2, 0x3f594d89, 0x3f2fac87, 0x3f064092,
+    0x3eba1245, 0x3e5016cd, 0x3d335c27, 0x3e1ac9dd, 0x3ea0a6f1, 0x3ef37edc,
+    0x3f22f6de, 0x3f4bfa4d, 0x3f74ca3e, 0x3f6298cf, 0x3f3a2e5b, 0x3f11f5e8,
+    0x3ed3ddf9, 0x3e84323c, 0x3dd39eaa, 0x3deb3986, 0x3e8ba34a, 0x3edc142f,
+    0x3f161103, 0x3f3de6e2, 0x3f658c2b, 0x3f72feac, 0x3f4bb92e, 0x3f24a2e9,
+    0x3efb76d9, 0x3eae048f, 0x3e41dc34, 0x3d219509, 0x3d50153e, 0x3e511b46,
+    0x3eb6ba2d, 0x3f024494, 0x3f28fdb9, 0x3f4f88f3, 0x3f75e6af, 0x3f63e8a7,
+    0x3f3de4a8, 0x3f180cea, 0x3ee4c20e, 0x3e99c134, 0x3e1e2cfc, 0x3c1824f4,
+    0x3de0bac6, 0x3e8436b1, 0x3ecfe62d, 0x3f0d9ef9, 0x3f331f66, 0x3f5874c1,
+    0x3f7d9f6c, 0x3f5d6037, 0x3f3889c9, 0x3f13dcea, 0x3edeb27d, 0x3e95fcd3,
+    0x3e1b303e, 0x3c3075cb, 0x3e0a7f24, 0x3e8eec6e, 0x3ed8462b, 0x3f10a6c1,
+    0x3f350197, 0x3f5933f1, 0x3f7d3e29, 0x3f5edf68, 0x3f3b246a, 0x3f179088,
+    0x3ee846d8, 0x3ea1b983, 0x3e36f0d8, 0x3d2c1773, 0x3e048260, 0x3e89b72b,
+    0x3ed0def0, 0x3f0bdc94, 0x3f2f233e, 0x3f5243ca, 0x3f753e89, 0x3f67ec34,
+    0x3f453c1d, 0x3f22b0e2, 0x3f004a36, 0x3ebc0f98, 0x3e6fa55d, 0x3dcf7467,
+    0x3dc09e5f, 0x3e6b0f8d, 0x3eba9e3c, 0x3eff6b94, 0x3f21f834, 0x3f4416a9,
+    0x3f661173, 0x3f781723, 0x3f5662cf, 0x3f34d14a, 0x3f13624c, 0x3ee42b1a,
+    0x3ea1d591, 0x3e3f86e3, 0x3d6fa1a1, 0x3cfd1ba9, 0x3e2674c3, 0x3e965d6c,
+    0x3ed93b69, 0x3f0dea73, 0x3f2f1538, 0x3f501e47, 0x3f7105e6, 0x3f6e33a9,
+    0x3f4d8e22, 0x3f2d0944, 0x3f0ca4cd, 0x3ed8c0fd, 0x3e98782f, 0x3e30dd66,
+    0x3d452061, 0x3d8e62bc, 0x3e49c779, 0x3ea5ed78, 0x3ee6b665, 0x3f139f81,
+    0x3f33c3e8, 0x3f53c8a7, 0x3f73adfa, 0x3f6c8be0, 0x3f4ce4ab, 0x3f2d5c2a,
+    0x3f0df223, 0x3edd4cb5, 0x3e9ef12d, 0x3e41a276, 0x3d8bb1ba, 0x3d9ba0ff,
+    0x3e4c6d54, 0x3ea547ab, 0x3ee41bba, 0x3f1159a6, 0x3f30876a, 0x3f4f9762,
+    0x3f6e89c9, 0x3f72a12b, 0x3f53e942, 0x3f354e46, 0x3f16cffe, 0x3ef0dc6f,
+    0x3eb45177, 0x3e6ffd59, 0x3def8e9c
+
+};
+
+const int32_t window_hex[] = {
+    0x00000000, 0x398b03f6, 0x3a61d1c5, 0x3ae0ee32, 0x3b37623a, 0x3b85f871,
+    0x3bb69d19, 0x3bed453b, 0x3c14d40b, 0x3c35c45b, 0x3c59595d, 0x3c7f7c1d,
+    0x3c940c13, 0x3ca98d81, 0x3cc039eb, 0x3cd8098d, 0x3cf0f52e, 0x3d057b06,
+    0x3d1302e6, 0x3d210f33, 0x3d2f9d0e, 0x3d3ea9ba, 0x3d4e3293, 0x3d5e3510,
+    0x3d6eaebd, 0x3d7f9d38, 0x3d887f19, 0x3d9167b5, 0x3d9a8756, 0x3da3dce9,
+    0x3dad675d, 0x3db725ab, 0x3dc116cc, 0x3dcb39bf, 0x3dd58d86, 0x3de01126,
+    0x3deac3a7, 0x3df5a413, 0x3e0058bb, 0x3e05f571, 0x3e0ba7b2, 0x3e116f08,
+    0x3e174afe, 0x3e1d3b1c, 0x3e233ef0, 0x3e295605, 0x3e2f7fe7, 0x3e35bc23,
+    0x3e3c0a46, 0x3e4269de, 0x3e48da79, 0x3e4f5ba5, 0x3e55ecf2, 0x3e5c8ded,
+    0x3e633e26, 0x3e69fd2c, 0x3e70ca8f, 0x3e77a5de, 0x3e7e8eaa, 0x3e82c241,
+    0x3e86437c, 0x3e89cacd, 0x3e8d57fc, 0x3e90ead3, 0x3e948319, 0x3e982097,
+    0x3e9bc316, 0x3e9f6a5d, 0x3ea31636, 0x3ea6c66a, 0x3eaa7ac0, 0x3eae3303,
+    0x3eb1eefa, 0x3eb5ae6f, 0x3eb9712a, 0x3ebd36f6, 0x3ec0ff9b, 0x3ec4cae2,
+    0x3ec89895, 0x3ecc687d, 0x3ed03a64, 0x3ed40e13, 0x3ed7e354, 0x3edbb9f2,
+    0x3edf91b5, 0x3ee36a69, 0x3ee743d7, 0x3eeb1dca, 0x3eeef80c, 0x3ef2d267,
+    0x3ef6aca8, 0x3efa8698, 0x3efe6002, 0x3f011c59, 0x3f03083a, 0x3f04f389,
+    0x3f06de2d, 0x3f08c80b, 0x3f0ab10a, 0x3f0c990f, 0x3f0e8001, 0x3f1065c6,
+    0x3f124a45, 0x3f142d65, 0x3f160f0c, 0x3f17ef21, 0x3f19cd8b, 0x3f1baa32,
+    0x3f1d84fb, 0x3f1f5dd0, 0x3f213498, 0x3f230939, 0x3f24db9d, 0x3f26abaa,
+    0x3f28794a, 0x3f2a4464, 0x3f2c0ce1, 0x3f2dd2a9, 0x3f2f95a6, 0x3f3155bf,
+    0x3f3312e0, 0x3f34ccef, 0x3f3683d8, 0x3f383784, 0x3f39e7dd, 0x3f3b94cc,
+    0x3f3d3e3c, 0x3f3ee418, 0x3f40864a, 0x3f4224bd, 0x3f43bf5c, 0x3f455613,
+    0x3f46e8cc, 0x3f487774, 0x3f4a01f6, 0x3f4b883f, 0x3f4d0a3b, 0x3f4e87d6,
+    0x3f5000fe, 0x3f5175a0, 0x3f52e5a9, 0x3f545106, 0x3f55b7a5, 0x3f571975,
+    0x3f587664, 0x3f59ce60, 0x3f5b2158, 0x3f5c6f3b, 0x3f5db7f9, 0x3f5efb80,
+    0x3f6039c2, 0x3f6172af, 0x3f62a636, 0x3f63d448, 0x3f64fcd6, 0x3f661fd3,
+    0x3f673d2e, 0x3f6854db, 0x3f6966ca, 0x3f6a72ef, 0x3f6b793d, 0x3f6c79a5,
+    0x3f6d741d, 0x3f6e6896, 0x3f6f5706, 0x3f703f5f, 0x3f712198, 0x3f71fda4,
+    0x3f72d379, 0x3f73a30c, 0x3f746c52, 0x3f752f43, 0x3f75ebd4, 0x3f76a1fc,
+    0x3f7751b2, 0x3f77faee, 0x3f789da6, 0x3f7939d4, 0x3f79cf6e, 0x3f7a5e6f,
+    0x3f7ae6cf, 0x3f7b6886, 0x3f7be38f, 0x3f7c57e4, 0x3f7cc57f, 0x3f7d2c5b,
+    0x3f7d8c72, 0x3f7de5bf, 0x3f7e3840, 0x3f7e83ee, 0x3f7ec8c7, 0x3f7f06c7,
+    0x3f7f3deb, 0x3f7f6e31, 0x3f7f9795, 0x3f7fba17, 0x3f7fd5b4, 0x3f7fea6b,
+    0x3f7ff83b, 0x3f7fff23, 0x3f7fff23, 0x3f7ff83b, 0x3f7fea6b, 0x3f7fd5b4,
+    0x3f7fba17, 0x3f7f9795, 0x3f7f6e31, 0x3f7f3deb, 0x3f7f06c7, 0x3f7ec8c7,
+    0x3f7e83ee, 0x3f7e3840, 0x3f7de5bf, 0x3f7d8c72, 0x3f7d2c5b, 0x3f7cc57f,
+    0x3f7c57e4, 0x3f7be38f, 0x3f7b6886, 0x3f7ae6cf, 0x3f7a5e6f, 0x3f79cf6e,
+    0x3f7939d4, 0x3f789da6, 0x3f77faee, 0x3f7751b2, 0x3f76a1fc, 0x3f75ebd4,
+    0x3f752f43, 0x3f746c52, 0x3f73a30c, 0x3f72d379, 0x3f71fda4, 0x3f712198,
+    0x3f703f5f, 0x3f6f5706, 0x3f6e6896, 0x3f6d741d, 0x3f6c79a5, 0x3f6b793d,
+    0x3f6a72ef, 0x3f6966ca, 0x3f6854db, 0x3f673d2e, 0x3f661fd3, 0x3f64fcd6,
+    0x3f63d448, 0x3f62a636, 0x3f6172af, 0x3f6039c2, 0x3f5efb80, 0x3f5db7f9,
+    0x3f5c6f3b, 0x3f5b2158, 0x3f59ce60, 0x3f587664, 0x3f571975, 0x3f55b7a5,
+    0x3f545106, 0x3f52e5a9, 0x3f5175a0, 0x3f5000fe, 0x3f4e87d6, 0x3f4d0a3b,
+    0x3f4b883f, 0x3f4a01f6, 0x3f487774, 0x3f46e8cc, 0x3f455613, 0x3f43bf5c,
+    0x3f4224bd, 0x3f40864a, 0x3f3ee418, 0x3f3d3e3c, 0x3f3b94cc, 0x3f39e7dd,
+    0x3f383784, 0x3f3683d8, 0x3f34ccef, 0x3f3312e0, 0x3f3155bf, 0x3f2f95a6,
+    0x3f2dd2a9, 0x3f2c0ce1, 0x3f2a4464, 0x3f28794a, 0x3f26abaa, 0x3f24db9d,
+    0x3f230939, 0x3f213498, 0x3f1f5dd0, 0x3f1d84fb, 0x3f1baa32, 0x3f19cd8b,
+    0x3f17ef21, 0x3f160f0c, 0x3f142d65, 0x3f124a45, 0x3f1065c6, 0x3f0e8001,
+    0x3f0c990f, 0x3f0ab10a, 0x3f08c80b, 0x3f06de2d, 0x3f04f389, 0x3f03083a,
+    0x3f011c59, 0x3efe6002, 0x3efa8698, 0x3ef6aca8, 0x3ef2d267, 0x3eeef80c,
+    0x3eeb1dca, 0x3ee743d7, 0x3ee36a69, 0x3edf91b5, 0x3edbb9f2, 0x3ed7e354,
+    0x3ed40e13, 0x3ed03a64, 0x3ecc687d, 0x3ec89895, 0x3ec4cae2, 0x3ec0ff9b,
+    0x3ebd36f6, 0x3eb9712a, 0x3eb5ae6f, 0x3eb1eefa, 0x3eae3303, 0x3eaa7ac0,
+    0x3ea6c66a, 0x3ea31636, 0x3e9f6a5d, 0x3e9bc316, 0x3e982097, 0x3e948319,
+    0x3e90ead3, 0x3e8d57fc, 0x3e89cacd, 0x3e86437c, 0x3e82c241, 0x3e7e8eaa,
+    0x3e77a5de, 0x3e70ca8f, 0x3e69fd2c, 0x3e633e26, 0x3e5c8ded, 0x3e55ecf2,
+    0x3e4f5ba5, 0x3e48da79, 0x3e4269de, 0x3e3c0a46, 0x3e35bc23, 0x3e2f7fe7,
+    0x3e295605, 0x3e233ef0, 0x3e1d3b1c, 0x3e174afe, 0x3e116f08, 0x3e0ba7b2,
+    0x3e05f571, 0x3e0058bb, 0x3df5a413, 0x3deac3a7, 0x3de01126, 0x3dd58d86,
+    0x3dcb39bf, 0x3dc116cc, 0x3db725ab, 0x3dad675d, 0x3da3dce9, 0x3d9a8756,
+    0x3d9167b5, 0x3d887f19, 0x3d7f9d38, 0x3d6eaebd, 0x3d5e3510, 0x3d4e3293,
+    0x3d3ea9ba, 0x3d2f9d0e, 0x3d210f33, 0x3d1302e6, 0x3d057b06, 0x3cf0f52e,
+    0x3cd8098d, 0x3cc039eb, 0x3ca98d81, 0x3c940c13, 0x3c7f7c1d, 0x3c59595d,
+    0x3c35c45b, 0x3c14d40b, 0x3bed453b, 0x3bb69d19, 0x3b85f871, 0x3b37623a,
+    0x3ae0ee32, 0x3a61d1c5, 0x398b03f6, 0x00000000
+
+};
+
+const int32_t window_hamm_hex[] = {
+    0x3da3d70a, 0x3da3f4f1, 0x3da44ea4, 0x3da4e41d, 0x3da5b554, 0x3da6c239,
+    0x3da80abd, 0x3da98ecb, 0x3dab4e4a, 0x3dad491d, 0x3daf7f25, 0x3db1f03d,
+    0x3db49c3e, 0x3db782fd, 0x3dbaa449, 0x3dbdfff1, 0x3dc195be, 0x3dc56575,
+    0x3dc96ed9, 0x3dcdb1a8, 0x3dd22d9d, 0x3dd6e26e, 0x3ddbcfd0, 0x3de0f572,
+    0x3de65301, 0x3debe825, 0x3df1b484, 0x3df7b7c0, 0x3dfdf176, 0x3e0230a1,
+    0x3e05835d, 0x3e08f0ba, 0x3e0c7880, 0x3e101a75, 0x3e13d65f, 0x3e17ac00,
+    0x3e1b9b1b, 0x3e1fa36f, 0x3e23c4bc, 0x3e27febd, 0x3e2c512e, 0x3e30bbc9,
+    0x3e353e46, 0x3e39d85c, 0x3e3e89c0, 0x3e435226, 0x3e483140, 0x3e4d26be,
+    0x3e523251, 0x3e5753a7, 0x3e5c8a6b, 0x3e61d64a, 0x3e6736ec, 0x3e6cabfc,
+    0x3e72351f, 0x3e77d1fd, 0x3e7d8239, 0x3e81a2bc, 0x3e848dae, 0x3e8781c3,
+    0x3e8a7eca, 0x3e8d8495, 0x3e9092f0, 0x3e93a9ab, 0x3e96c894, 0x3e99ef77,
+    0x3e9d1e22, 0x3ea05460, 0x3ea391ff, 0x3ea6d6c8, 0x3eaa2286, 0x3ead7505,
+    0x3eb0ce0f, 0x3eb42d6c, 0x3eb792e6, 0x3ebafe46, 0x3ebe6f54, 0x3ec1e5d9,
+    0x3ec5619c, 0x3ec8e264, 0x3ecc67f8, 0x3ecff220, 0x3ed380a2, 0x3ed71344,
+    0x3edaa9cb, 0x3ede43fe, 0x3ee1e1a3, 0x3ee5827d, 0x3ee92653, 0x3eeccce9,
+    0x3ef07604, 0x3ef42168, 0x3ef7ceda, 0x3efb7e1d, 0x3eff2ef7, 0x3f017096,
+    0x3f034a3f, 0x3f052459, 0x3f06fec5, 0x3f08d967, 0x3f0ab41f, 0x3f0c8ed0,
+    0x3f0e695b, 0x3f1043a2, 0x3f121d87, 0x3f13f6ec, 0x3f15cfb4, 0x3f17a7bf,
+    0x3f197ef0, 0x3f1b5529, 0x3f1d2a4d, 0x3f1efe3d, 0x3f20d0db, 0x3f22a20b,
+    0x3f2471ae, 0x3f263fa8, 0x3f280bda, 0x3f29d628, 0x3f2b9e74, 0x3f2d64a2,
+    0x3f2f2895, 0x3f30ea30, 0x3f32a956, 0x3f3465ec, 0x3f361fd4, 0x3f37d6f3,
+    0x3f398b2d, 0x3f3b3c66, 0x3f3cea83, 0x3f3e9569, 0x3f403cfb, 0x3f41e121,
+    0x3f4381be, 0x3f451eb8, 0x3f46b7f6, 0x3f484d5d, 0x3f49ded3, 0x3f4b6c3f,
+    0x3f4cf588, 0x3f4e7a94, 0x3f4ffb4c, 0x3f517796, 0x3f52ef5a, 0x3f546282,
+    0x3f55d0f4, 0x3f573a9a, 0x3f589f5d, 0x3f59ff26, 0x3f5b59df, 0x3f5caf72,
+    0x3f5dffc9, 0x3f5f4acf, 0x3f60906f, 0x3f61d093, 0x3f630b29, 0x3f64401b,
+    0x3f656f57, 0x3f6698c9, 0x3f67bc5d, 0x3f68da03, 0x3f69f1a6, 0x3f6b0337,
+    0x3f6c0ea3, 0x3f6d13d9, 0x3f6e12c9, 0x3f6f0b62, 0x3f6ffd95, 0x3f70e953,
+    0x3f71ce8c, 0x3f72ad32, 0x3f738537, 0x3f74568d, 0x3f752127, 0x3f75e4f8,
+    0x3f76a1f3, 0x3f77580d, 0x3f780739, 0x3f78af6e, 0x3f79509f, 0x3f79eac3,
+    0x3f7a7dd1, 0x3f7b09be, 0x3f7b8e83, 0x3f7c0c15, 0x3f7c826e, 0x3f7cf187,
+    0x3f7d5957, 0x3f7db9d8, 0x3f7e1305, 0x3f7e64d7, 0x3f7eaf4a, 0x3f7ef258,
+    0x3f7f2dfe, 0x3f7f6237, 0x3f7f8f00, 0x3f7fb457, 0x3f7fd239, 0x3f7fe8a4,
+    0x3f7ff797, 0x3f7fff11, 0x3f7fff11, 0x3f7ff797, 0x3f7fe8a4, 0x3f7fd239,
+    0x3f7fb457, 0x3f7f8f00, 0x3f7f6237, 0x3f7f2dfe, 0x3f7ef258, 0x3f7eaf4a,
+    0x3f7e64d7, 0x3f7e1305, 0x3f7db9d8, 0x3f7d5957, 0x3f7cf187, 0x3f7c826e,
+    0x3f7c0c15, 0x3f7b8e83, 0x3f7b09be, 0x3f7a7dd1, 0x3f79eac3, 0x3f79509f,
+    0x3f78af6e, 0x3f780739, 0x3f77580d, 0x3f76a1f3, 0x3f75e4f8, 0x3f752127,
+    0x3f74568d, 0x3f738537, 0x3f72ad32, 0x3f71ce8c, 0x3f70e953, 0x3f6ffd95,
+    0x3f6f0b62, 0x3f6e12c9, 0x3f6d13d9, 0x3f6c0ea3, 0x3f6b0337, 0x3f69f1a6,
+    0x3f68da03, 0x3f67bc5d, 0x3f6698c9, 0x3f656f57, 0x3f64401b, 0x3f630b29,
+    0x3f61d093, 0x3f60906f, 0x3f5f4acf, 0x3f5dffc9, 0x3f5caf72, 0x3f5b59df,
+    0x3f59ff26, 0x3f589f5d, 0x3f573a9a, 0x3f55d0f4, 0x3f546282, 0x3f52ef5a,
+    0x3f517796, 0x3f4ffb4c, 0x3f4e7a94, 0x3f4cf588, 0x3f4b6c3f, 0x3f49ded3,
+    0x3f484d5d, 0x3f46b7f6, 0x3f451eb8, 0x3f4381be, 0x3f41e121, 0x3f403cfb,
+    0x3f3e9569, 0x3f3cea83, 0x3f3b3c66, 0x3f398b2d, 0x3f37d6f3, 0x3f361fd4,
+    0x3f3465ec, 0x3f32a956, 0x3f30ea30, 0x3f2f2895, 0x3f2d64a2, 0x3f2b9e74,
+    0x3f29d628, 0x3f280bda, 0x3f263fa8, 0x3f2471ae, 0x3f22a20b, 0x3f20d0db,
+    0x3f1efe3d, 0x3f1d2a4d, 0x3f1b5529, 0x3f197ef0, 0x3f17a7bf, 0x3f15cfb4,
+    0x3f13f6ec, 0x3f121d87, 0x3f1043a2, 0x3f0e695b, 0x3f0c8ed0, 0x3f0ab41f,
+    0x3f08d967, 0x3f06fec5, 0x3f052459, 0x3f034a3f, 0x3f017096, 0x3eff2ef7,
+    0x3efb7e1d, 0x3ef7ceda, 0x3ef42168, 0x3ef07604, 0x3eeccce9, 0x3ee92653,
+    0x3ee5827d, 0x3ee1e1a3, 0x3ede43fe, 0x3edaa9cb, 0x3ed71344, 0x3ed380a2,
+    0x3ecff220, 0x3ecc67f8, 0x3ec8e264, 0x3ec5619c, 0x3ec1e5d9, 0x3ebe6f54,
+    0x3ebafe46, 0x3eb792e6, 0x3eb42d6c, 0x3eb0ce0f, 0x3ead7505, 0x3eaa2286,
+    0x3ea6d6c8, 0x3ea391ff, 0x3ea05460, 0x3e9d1e22, 0x3e99ef77, 0x3e96c894,
+    0x3e93a9ab, 0x3e9092f0, 0x3e8d8495, 0x3e8a7eca, 0x3e8781c3, 0x3e848dae,
+    0x3e81a2bc, 0x3e7d8239, 0x3e77d1fd, 0x3e72351f, 0x3e6cabfc, 0x3e6736ec,
+    0x3e61d64a, 0x3e5c8a6b, 0x3e5753a7, 0x3e523251, 0x3e4d26be, 0x3e483140,
+    0x3e435226, 0x3e3e89c0, 0x3e39d85c, 0x3e353e46, 0x3e30bbc9, 0x3e2c512e,
+    0x3e27febd, 0x3e23c4bc, 0x3e1fa36f, 0x3e1b9b1b, 0x3e17ac00, 0x3e13d65f,
+    0x3e101a75, 0x3e0c7880, 0x3e08f0ba, 0x3e05835d, 0x3e0230a1, 0x3dfdf176,
+    0x3df7b7c0, 0x3df1b484, 0x3debe825, 0x3de65301, 0x3de0f572, 0x3ddbcfd0,
+    0x3dd6e26e, 0x3dd22d9d, 0x3dcdb1a8, 0x3dc96ed9, 0x3dc56575, 0x3dc195be,
+    0x3dbdfff1, 0x3dbaa449, 0x3db782fd, 0x3db49c3e, 0x3db1f03d, 0x3daf7f25,
+    0x3dad491d, 0x3dab4e4a, 0x3da98ecb, 0x3da80abd, 0x3da6c239, 0x3da5b554,
+    0x3da4e41d, 0x3da44ea4, 0x3da3f4f1, 0x3da3d70a
+
+};
+
+const int global_cmvn_mean_hex[] = {
+    0x413d6566, 0x4147923f, 0x4156ab15, 0x41613d12, 0x416b155b, 0x41722783,
+    0x4176cd05, 0x4178532a, 0x417aa3c3, 0x417aed19, 0x417d4d2c, 0x417e6abb,
+    0x41805848, 0x418122ab, 0x41812b23, 0x418161a8, 0x41810ef9, 0x4180863a,
+    0x41815d8f, 0x417ff8b2, 0x417de2aa, 0x4180a5f2, 0x417e8bd1, 0x418041ac,
+    0x417f2d60, 0x4180487f, 0x417eb835, 0x418018d8, 0x417ef8c1, 0x417ea302,
+    0x417f30cf, 0x417ea0bb, 0x417ebac2, 0x417faab6, 0x417fca4d, 0x41805e45,
+    0x4180e308, 0x4180ef3e, 0x418109fc, 0x4180afa3, 0x418113e2, 0x4180c915,
+    0x41819f86, 0x418190bf, 0x418220bd, 0x4182f2e5, 0x4183e1c7, 0x41843eec,
+    0x4184b066, 0x418574db, 0x41852611, 0x4184fc81, 0x41851b2a, 0x4185a1c7,
+    0x41861152, 0x41868c28, 0x41871930, 0x41871f83, 0x41868893, 0x4185d919,
+    0x4185664b, 0x418480a6, 0x41840e3a, 0x41836ace, 0x4182b217, 0x4181cb79,
+    0x4180fb13, 0x418098b9, 0x41805ded, 0x417ff69a, 0x417f49bd, 0x417ecef8,
+    0x417e286c, 0x417d9135, 0x417cfff4, 0x417ca8f7, 0x417b2e8f, 0x41773788,
+    0x4170b095, 0x4167417f};
+
+const int global_cmvn_std_hex[] = {
+    0x4040335e, 0x405235d3, 0x40589be4, 0x4054261f, 0x40544ba2, 0x40575418,
+    0x405b6528, 0x40617999, 0x40605fcf, 0x405c9c6d, 0x40590796, 0x405899fc,
+    0x405810b8, 0x40587c40, 0x40592b5e, 0x4057fb12, 0x4057028b, 0x405515d7,
+    0x4053d714, 0x405418c7, 0x405536bc, 0x4052f54e, 0x4052d382, 0x4051201d,
+    0x4050a8d2, 0x4050857f, 0x404ffe85, 0x4050a0da, 0x40517a8a, 0x40508862,
+    0x40504f68, 0x404f3159, 0x404f0930, 0x404e8a2e, 0x404e7383, 0x404eb185,
+    0x404edaa9, 0x404efed2, 0x404ea8f4, 0x404f6d0d, 0x404ee9d9, 0x404f4cca,
+    0x404fb13f, 0x405051c5, 0x40503f5e, 0x4050df6e, 0x4052974e, 0x4053d421,
+    0x40544d48, 0x40544ec8, 0x40550e57, 0x40558287, 0x4055d122, 0x4056b22a,
+    0x4058ea5c, 0x405acbc3, 0x405a89e7, 0x405a88ed, 0x405afadb, 0x405a1c60,
+    0x405a6f46, 0x405b0a24, 0x405b5f44, 0x405cc0a9, 0x405d984b, 0x405ef9b8,
+    0x4061178a, 0x406262bf, 0x40644904, 0x40660b20, 0x4067f7f1, 0x406a35e5,
+    0x406c1e97, 0x406e16a9, 0x406eadb1, 0x406d0cba, 0x406d9ca0, 0x406f5a14,
+    0x406e84a7, 0x406cd985};
+
+const int global_cmvn_mean_online_hex[] = {
+
+    0x413d5d27, 0x414785ae, 0x4156986a, 0x41612a4e, 0x416b063e, 0x41721c9b,
+    0x4176c505, 0x41784b5b, 0x417a9575, 0x417adfb2, 0x417d4153, 0x417e611e,
+    0x41805288, 0x41811c27, 0x4181250c, 0x41815cd4, 0x41810b77, 0x4180817c,
+    0x41815881, 0x417feaf2, 0x417dd2bf, 0x41809f37, 0x417e7b47, 0x41803a6a,
+    0x417f1ff4, 0x41804382, 0x417ead10, 0x41801220, 0x417eeb28, 0x417e9801,
+    0x417f26b9, 0x417e95f9, 0x417eac06, 0x417f9aa5, 0x417fbb16, 0x41805651,
+    0x4180daaa, 0x4180e84c, 0x41810566, 0x4180ab2c, 0x418111b0, 0x4180c6cc,
+    0x41819e27, 0x418190cc, 0x4182205c, 0x4182f265, 0x4183e1a2, 0x41844012,
+    0x4184b0cd, 0x41857447, 0x418527f7, 0x4184fdc6, 0x41851ad2, 0x4185a148,
+    0x41860f8b, 0x41868888, 0x418712e4, 0x41871702, 0x41867ec3, 0x4185cc48,
+    0x418559b4, 0x41847855, 0x418408f4, 0x418368f4, 0x4182b718, 0x4181d76d,
+    0x41810e52, 0x4180b204, 0x418078a4, 0x41801179, 0x417f5579, 0x417e93b7,
+    0x417d6f2c, 0x417c1a0b, 0x417a6c7a, 0x41787d18, 0x4174eceb, 0x416e3ed3,
+    0x41644af8, 0x41566dd4
+
+};
+
+const int global_cmvn_std_online_hex[] = {
+
+    0x40408fdd, 0x405293b6, 0x4058f2d2, 0x40546ddb, 0x4054984c, 0x4057971b,
+    0x405ba086, 0x4061afa7, 0x4060a24c, 0x405cbb7e, 0x405923f7, 0x4058c91f,
+    0x40585cf3, 0x4058c22a, 0x40594960, 0x405824a6, 0x405703f3, 0x40556377,
+    0x4053e02d, 0x40540a7e, 0x405553c7, 0x4052ead5, 0x4052d23d, 0x40510308,
+    0x4050a2f3, 0x40505b81, 0x404fed20, 0x4050a372, 0x40515196, 0x40504810,
+    0x40501fdd, 0x404f2225, 0x404f0931, 0x404e8a2b, 0x404e773b, 0x404ea782,
+    0x404ee17d, 0x404ef49c, 0x404e884d, 0x404f696b, 0x404edd0e, 0x404f23cc,
+    0x404f74d4, 0x40501e89, 0x405009f3, 0x4050c422, 0x4052902b, 0x4053987c,
+    0x40542997, 0x40543695, 0x4054cbef, 0x40553947, 0x4055ab7c, 0x4056887c,
+    0x4058b710, 0x405a8d28, 0x405a6a27, 0x405a6b3b, 0x405ac8d3, 0x405a031d,
+    0x405a2158, 0x405abb1b, 0x405b1350, 0x405c98c0, 0x405d5cf9, 0x405ead5b,
+    0x40609748, 0x4061dfb9, 0x4063aa9f, 0x40655831, 0x40671a35, 0x40694bf5,
+    0x406b1f59, 0x406cb49b, 0x406cf19e, 0x406b592b, 0x406b757c, 0x406c866d,
+    0x406ac24f, 0x406678d9
+
+};
+
+const unsigned int paraformer_cmvn_mean_hex[] = {
+
+    0xc104fd75, 0xc1099d56, 0xc119dad7, 0xc126f9a7, 0xc133681f, 0xc13e221f,
+    0xc145cc83, 0xc14a3166, 0xc14e1bda, 0xc14d4a62, 0xc14e41a9, 0xc14f4e7b,
+    0xc153297e, 0xc1567ee5, 0xc157dbab, 0xc158dfa4, 0xc158e6f9, 0xc1584e70,
+    0xc15aecea, 0xc15886b8, 0xc156bcb4, 0xc15a7ba9, 0xc1581d34, 0xc15c0a48,
+    0xc15c463f, 0xc15dfc3b, 0xc15bb28b, 0xc15b4413, 0xc158f8c0, 0xc1588ede,
+    0xc158c880, 0xc158ff19, 0xc159815a, 0xc159ed72, 0xc15a458d, 0xc15a93d3,
+    0xc15a06ec, 0xc15953d8, 0xc1592e92, 0xc1579518, 0xc1587d76, 0xc157bc56,
+    0xc159c47c, 0xc15a5ac4, 0xc15b7286, 0xc15cab60, 0xc15e7f8d, 0xc1607ee5,
+    0xc162e9ad, 0xc165bdb0, 0xc167bf3e, 0xc169a0a5, 0xc16b4b68, 0xc16d5682,
+    0xc16ebd51, 0xc170197a, 0xc170d1cc, 0xc1707fc1, 0xc16fd830, 0xc16ec4b1,
+    0xc16de888, 0xc16d3b06, 0xc16cc155, 0xc16c4e31, 0xc16b6abe, 0xc169cde8,
+    0xc1684578, 0xc166c2a4, 0xc165d326, 0xc164df46, 0xc163b4ad, 0xc1632d19,
+    0xc162a94a, 0xc16280fc, 0xc161ae3e, 0xc15fec42, 0xc15cbadc, 0xc15664c3,
+    0xc14c6d5d, 0xc13b64ae, 0xc104fd75, 0xc1099d56, 0xc119dad7, 0xc126f9a7,
+    0xc133681f, 0xc13e221f, 0xc145cc83, 0xc14a3166, 0xc14e1bda, 0xc14d4a62,
+    0xc14e41a9, 0xc14f4e7b, 0xc153297e, 0xc1567ee5, 0xc157dbab, 0xc158dfa4,
+    0xc158e6f9, 0xc1584e70, 0xc15aecea, 0xc15886b8, 0xc156bcb4, 0xc15a7ba9,
+    0xc1581d34, 0xc15c0a48, 0xc15c463f, 0xc15dfc3b, 0xc15bb28b, 0xc15b4413,
+    0xc158f8c0, 0xc1588ede, 0xc158c880, 0xc158ff19, 0xc159815a, 0xc159ed72,
+    0xc15a458d, 0xc15a93d3, 0xc15a06ec, 0xc15953d8, 0xc1592e92, 0xc1579518,
+    0xc1587d76, 0xc157bc56, 0xc159c47c, 0xc15a5ac4, 0xc15b7286, 0xc15cab60,
+    0xc15e7f8d, 0xc1607ee5, 0xc162e9ad, 0xc165bdb0, 0xc167bf3e, 0xc169a0a5,
+    0xc16b4b68, 0xc16d5682, 0xc16ebd51, 0xc170197a, 0xc170d1cc, 0xc1707fc1,
+    0xc16fd830, 0xc16ec4b1, 0xc16de888, 0xc16d3b06, 0xc16cc155, 0xc16c4e31,
+    0xc16b6abe, 0xc169cde8, 0xc1684578, 0xc166c2a4, 0xc165d326, 0xc164df46,
+    0xc163b4ad, 0xc1632d19, 0xc162a94a, 0xc16280fc, 0xc161ae3e, 0xc15fec42,
+    0xc15cbadc, 0xc15664c3, 0xc14c6d5d, 0xc13b64ae, 0xc104fd75, 0xc1099d56,
+    0xc119dad7, 0xc126f9a7, 0xc133681f, 0xc13e221f, 0xc145cc83, 0xc14a3166,
+    0xc14e1bda, 0xc14d4a62, 0xc14e41a9, 0xc14f4e7b, 0xc153297e, 0xc1567ee5,
+    0xc157dbab, 0xc158dfa4, 0xc158e6f9, 0xc1584e70, 0xc15aecea, 0xc15886b8,
+    0xc156bcb4, 0xc15a7ba9, 0xc1581d34, 0xc15c0a48, 0xc15c463f, 0xc15dfc3b,
+    0xc15bb28b, 0xc15b4413, 0xc158f8c0, 0xc1588ede, 0xc158c880, 0xc158ff19,
+    0xc159815a, 0xc159ed72, 0xc15a458d, 0xc15a93d3, 0xc15a06ec, 0xc15953d8,
+    0xc1592e92, 0xc1579518, 0xc1587d76, 0xc157bc56, 0xc159c47c, 0xc15a5ac4,
+    0xc15b7286, 0xc15cab60, 0xc15e7f8d, 0xc1607ee5, 0xc162e9ad, 0xc165bdb0,
+    0xc167bf3e, 0xc169a0a5, 0xc16b4b68, 0xc16d5682, 0xc16ebd51, 0xc170197a,
+    0xc170d1cc, 0xc1707fc1, 0xc16fd830, 0xc16ec4b1, 0xc16de888, 0xc16d3b06,
+    0xc16cc155, 0xc16c4e31, 0xc16b6abe, 0xc169cde8, 0xc1684578, 0xc166c2a4,
+    0xc165d326, 0xc164df46, 0xc163b4ad, 0xc1632d19, 0xc162a94a, 0xc16280fc,
+    0xc161ae3e, 0xc15fec42, 0xc15cbadc, 0xc15664c3, 0xc14c6d5d, 0xc13b64ae,
+    0xc104fd75, 0xc1099d56, 0xc119dad7, 0xc126f9a7, 0xc133681f, 0xc13e221f,
+    0xc145cc83, 0xc14a3166, 0xc14e1bda, 0xc14d4a62, 0xc14e41a9, 0xc14f4e7b,
+    0xc153297e, 0xc1567ee5, 0xc157dbab, 0xc158dfa4, 0xc158e6f9, 0xc1584e70,
+    0xc15aecea, 0xc15886b8, 0xc156bcb4, 0xc15a7ba9, 0xc1581d34, 0xc15c0a48,
+    0xc15c463f, 0xc15dfc3b, 0xc15bb28b, 0xc15b4413, 0xc158f8c0, 0xc1588ede,
+    0xc158c880, 0xc158ff19, 0xc159815a, 0xc159ed72, 0xc15a458d, 0xc15a93d3,
+    0xc15a06ec, 0xc15953d8, 0xc1592e92, 0xc1579518, 0xc1587d76, 0xc157bc56,
+    0xc159c47c, 0xc15a5ac4, 0xc15b7286, 0xc15cab60, 0xc15e7f8d, 0xc1607ee5,
+    0xc162e9ad, 0xc165bdb0, 0xc167bf3e, 0xc169a0a5, 0xc16b4b68, 0xc16d5682,
+    0xc16ebd51, 0xc170197a, 0xc170d1cc, 0xc1707fc1, 0xc16fd830, 0xc16ec4b1,
+    0xc16de888, 0xc16d3b06, 0xc16cc155, 0xc16c4e31, 0xc16b6abe, 0xc169cde8,
+    0xc1684578, 0xc166c2a4, 0xc165d326, 0xc164df46, 0xc163b4ad, 0xc1632d19,
+    0xc162a94a, 0xc16280fc, 0xc161ae3e, 0xc15fec42, 0xc15cbadc, 0xc15664c3,
+    0xc14c6d5d, 0xc13b64ae, 0xc104fd75, 0xc1099d56, 0xc119dad7, 0xc126f9a7,
+    0xc133681f, 0xc13e221f, 0xc145cc83, 0xc14a3166, 0xc14e1bda, 0xc14d4a62,
+    0xc14e41a9, 0xc14f4e7b, 0xc153297e, 0xc1567ee5, 0xc157dbab, 0xc158dfa4,
+    0xc158e6f9, 0xc1584e70, 0xc15aecea, 0xc15886b8, 0xc156bcb4, 0xc15a7ba9,
+    0xc1581d34, 0xc15c0a48, 0xc15c463f, 0xc15dfc3b, 0xc15bb28b, 0xc15b4413,
+    0xc158f8c0, 0xc1588ede, 0xc158c880, 0xc158ff19, 0xc159815a, 0xc159ed72,
+    0xc15a458d, 0xc15a93d3, 0xc15a06ec, 0xc15953d8, 0xc1592e92, 0xc1579518,
+    0xc1587d76, 0xc157bc56, 0xc159c47c, 0xc15a5ac4, 0xc15b7286, 0xc15cab60,
+    0xc15e7f8d, 0xc1607ee5, 0xc162e9ad, 0xc165bdb0, 0xc167bf3e, 0xc169a0a5,
+    0xc16b4b68, 0xc16d5682, 0xc16ebd51, 0xc170197a, 0xc170d1cc, 0xc1707fc1,
+    0xc16fd830, 0xc16ec4b1, 0xc16de888, 0xc16d3b06, 0xc16cc155, 0xc16c4e31,
+    0xc16b6abe, 0xc169cde8, 0xc1684578, 0xc166c2a4, 0xc165d326, 0xc164df46,
+    0xc163b4ad, 0xc1632d19, 0xc162a94a, 0xc16280fc, 0xc161ae3e, 0xc15fec42,
+    0xc15cbadc, 0xc15664c3, 0xc14c6d5d, 0xc13b64ae, 0xc104fd75, 0xc1099d56,
+    0xc119dad7, 0xc126f9a7, 0xc133681f, 0xc13e221f, 0xc145cc83, 0xc14a3166,
+    0xc14e1bda, 0xc14d4a62, 0xc14e41a9, 0xc14f4e7b, 0xc153297e, 0xc1567ee5,
+    0xc157dbab, 0xc158dfa4, 0xc158e6f9, 0xc1584e70, 0xc15aecea, 0xc15886b8,
+    0xc156bcb4, 0xc15a7ba9, 0xc1581d34, 0xc15c0a48, 0xc15c463f, 0xc15dfc3b,
+    0xc15bb28b, 0xc15b4413, 0xc158f8c0, 0xc1588ede, 0xc158c880, 0xc158ff19,
+    0xc159815a, 0xc159ed72, 0xc15a458d, 0xc15a93d3, 0xc15a06ec, 0xc15953d8,
+    0xc1592e92, 0xc1579518, 0xc1587d76, 0xc157bc56, 0xc159c47c, 0xc15a5ac4,
+    0xc15b7286, 0xc15cab60, 0xc15e7f8d, 0xc1607ee5, 0xc162e9ad, 0xc165bdb0,
+    0xc167bf3e, 0xc169a0a5, 0xc16b4b68, 0xc16d5682, 0xc16ebd51, 0xc170197a,
+    0xc170d1cc, 0xc1707fc1, 0xc16fd830, 0xc16ec4b1, 0xc16de888, 0xc16d3b06,
+    0xc16cc155, 0xc16c4e31, 0xc16b6abe, 0xc169cde8, 0xc1684578, 0xc166c2a4,
+    0xc165d326, 0xc164df46, 0xc163b4ad, 0xc1632d19, 0xc162a94a, 0xc16280fc,
+    0xc161ae3e, 0xc15fec42, 0xc15cbadc, 0xc15664c3, 0xc14c6d5d, 0xc13b64ae,
+    0xc104fd75, 0xc1099d56, 0xc119dad7, 0xc126f9a7, 0xc133681f, 0xc13e221f,
+    0xc145cc83, 0xc14a3166, 0xc14e1bda, 0xc14d4a62, 0xc14e41a9, 0xc14f4e7b,
+    0xc153297e, 0xc1567ee5, 0xc157dbab, 0xc158dfa4, 0xc158e6f9, 0xc1584e70,
+    0xc15aecea, 0xc15886b8, 0xc156bcb4, 0xc15a7ba9, 0xc1581d34, 0xc15c0a48,
+    0xc15c463f, 0xc15dfc3b, 0xc15bb28b, 0xc15b4413, 0xc158f8c0, 0xc1588ede,
+    0xc158c880, 0xc158ff19, 0xc159815a, 0xc159ed72, 0xc15a458d, 0xc15a93d3,
+    0xc15a06ec, 0xc15953d8, 0xc1592e92, 0xc1579518, 0xc1587d76, 0xc157bc56,
+    0xc159c47c, 0xc15a5ac4, 0xc15b7286, 0xc15cab60, 0xc15e7f8d, 0xc1607ee5,
+    0xc162e9ad, 0xc165bdb0, 0xc167bf3e, 0xc169a0a5, 0xc16b4b68, 0xc16d5682,
+    0xc16ebd51, 0xc170197a, 0xc170d1cc, 0xc1707fc1, 0xc16fd830, 0xc16ec4b1,
+    0xc16de888, 0xc16d3b06, 0xc16cc155, 0xc16c4e31, 0xc16b6abe, 0xc169cde8,
+    0xc1684578, 0xc166c2a4, 0xc165d326, 0xc164df46, 0xc163b4ad, 0xc1632d19,
+    0xc162a94a, 0xc16280fc, 0xc161ae3e, 0xc15fec42, 0xc15cbadc, 0xc15664c3,
+    0xc14c6d5d, 0xc13b64ae};
+
+const unsigned int paraformer_cmvn_var_hex[] = {
+
+    0x40619618, 0x405fb77c, 0x405d3028, 0x405bef11, 0x405a189d, 0x4057aad5,
+    0x4054f9cc, 0x40518e8c, 0x404fffdd, 0x40510d0d, 0x4052400d, 0x4052bab0,
+    0x40526416, 0x40515cb8, 0x40506aee, 0x404fef8d, 0x404ff527, 0x40505b95,
+    0x4050d61c, 0x4051d0a5, 0x4052abd2, 0x4052f14b, 0x4053d196, 0x4054800d,
+    0x405545f2, 0x4055d71f, 0x40567588, 0x4056de4d, 0x40579b72, 0x40584d35,
+    0x4058cd2f, 0x40594731, 0x4059a53f, 0x405a00ed, 0x405a34c1, 0x405a406e,
+    0x405a1748, 0x405a0300, 0x405a1547, 0x405a66a7, 0x405a9be4, 0x405b04b2,
+    0x405b5754, 0x405b9189, 0x405b9016, 0x405b7a07, 0x405b63f9, 0x405b3f45,
+    0x405b0cb4, 0x405ac80b, 0x405ac1f7, 0x405abbd9, 0x405ac86a, 0x405ad72b,
+    0x405af2f0, 0x405ab465, 0x405a6364, 0x405a1350, 0x4059baa3, 0x4059911d,
+    0x40597921, 0x40595564, 0x40593b8d, 0x4059310f, 0x40594e46, 0x40599bae,
+    0x4059e703, 0x4059feec, 0x405a053a, 0x4059feaa, 0x4059d7a0, 0x40599386,
+    0x40592d0e, 0x4058ce4c, 0x40587335, 0x4058396a, 0x40584ee1, 0x4058925a,
+    0x40592f6d, 0x405a9f0a, 0x40619618, 0x405fb77c, 0x405d3028, 0x405bef11,
+    0x405a189d, 0x4057aad5, 0x4054f9cc, 0x40518e8c, 0x404fffdd, 0x40510d0d,
+    0x4052400d, 0x4052bab0, 0x40526416, 0x40515cb8, 0x40506aee, 0x404fef8d,
+    0x404ff527, 0x40505b95, 0x4050d61c, 0x4051d0a5, 0x4052abd2, 0x4052f14b,
+    0x4053d196, 0x4054800d, 0x405545f2, 0x4055d71f, 0x40567588, 0x4056de4d,
+    0x40579b72, 0x40584d35, 0x4058cd2f, 0x40594731, 0x4059a53f, 0x405a00ed,
+    0x405a34c1, 0x405a406e, 0x405a1748, 0x405a0300, 0x405a1547, 0x405a66a7,
+    0x405a9be4, 0x405b04b2, 0x405b5754, 0x405b9189, 0x405b9016, 0x405b7a07,
+    0x405b63f9, 0x405b3f45, 0x405b0cb4, 0x405ac80b, 0x405ac1f7, 0x405abbd9,
+    0x405ac86a, 0x405ad72b, 0x405af2f0, 0x405ab465, 0x405a6364, 0x405a1350,
+    0x4059baa3, 0x4059911d, 0x40597921, 0x40595564, 0x40593b8d, 0x4059310f,
+    0x40594e46, 0x40599bae, 0x4059e703, 0x4059feec, 0x405a053a, 0x4059feaa,
+    0x4059d7a0, 0x40599386, 0x40592d0e, 0x4058ce4c, 0x40587335, 0x4058396a,
+    0x40584ee1, 0x4058925a, 0x40592f6d, 0x405a9f0a, 0x40619618, 0x405fb77c,
+    0x405d3028, 0x405bef11, 0x405a189d, 0x4057aad5, 0x4054f9cc, 0x40518e8c,
+    0x404fffdd, 0x40510d0d, 0x4052400d, 0x4052bab0, 0x40526416, 0x40515cb8,
+    0x40506aee, 0x404fef8d, 0x404ff527, 0x40505b95, 0x4050d61c, 0x4051d0a5,
+    0x4052abd2, 0x4052f14b, 0x4053d196, 0x4054800d, 0x405545f2, 0x4055d71f,
+    0x40567588, 0x4056de4d, 0x40579b72, 0x40584d35, 0x4058cd2f, 0x40594731,
+    0x4059a53f, 0x405a00ed, 0x405a34c1, 0x405a406e, 0x405a1748, 0x405a0300,
+    0x405a1547, 0x405a66a7, 0x405a9be4, 0x405b04b2, 0x405b5754, 0x405b9189,
+    0x405b9016, 0x405b7a07, 0x405b63f9, 0x405b3f45, 0x405b0cb4, 0x405ac80b,
+    0x405ac1f7, 0x405abbd9, 0x405ac86a, 0x405ad72b, 0x405af2f0, 0x405ab465,
+    0x405a6364, 0x405a1350, 0x4059baa3, 0x4059911d, 0x40597921, 0x40595564,
+    0x40593b8d, 0x4059310f, 0x40594e46, 0x40599bae, 0x4059e703, 0x4059feec,
+    0x405a053a, 0x4059feaa, 0x4059d7a0, 0x40599386, 0x40592d0e, 0x4058ce4c,
+    0x40587335, 0x4058396a, 0x40584ee1, 0x4058925a, 0x40592f6d, 0x405a9f0a,
+    0x40619618, 0x405fb77c, 0x405d3028, 0x405bef11, 0x405a189d, 0x4057aad5,
+    0x4054f9cc, 0x40518e8c, 0x404fffdd, 0x40510d0d, 0x4052400d, 0x4052bab0,
+    0x40526416, 0x40515cb8, 0x40506aee, 0x404fef8d, 0x404ff527, 0x40505b95,
+    0x4050d61c, 0x4051d0a5, 0x4052abd2, 0x4052f14b, 0x4053d196, 0x4054800d,
+    0x405545f2, 0x4055d71f, 0x40567588, 0x4056de4d, 0x40579b72, 0x40584d35,
+    0x4058cd2f, 0x40594731, 0x4059a53f, 0x405a00ed, 0x405a34c1, 0x405a406e,
+    0x405a1748, 0x405a0300, 0x405a1547, 0x405a66a7, 0x405a9be4, 0x405b04b2,
+    0x405b5754, 0x405b9189, 0x405b9016, 0x405b7a07, 0x405b63f9, 0x405b3f45,
+    0x405b0cb4, 0x405ac80b, 0x405ac1f7, 0x405abbd9, 0x405ac86a, 0x405ad72b,
+    0x405af2f0, 0x405ab465, 0x405a6364, 0x405a1350, 0x4059baa3, 0x4059911d,
+    0x40597921, 0x40595564, 0x40593b8d, 0x4059310f, 0x40594e46, 0x40599bae,
+    0x4059e703, 0x4059feec, 0x405a053a, 0x4059feaa, 0x4059d7a0, 0x40599386,
+    0x40592d0e, 0x4058ce4c, 0x40587335, 0x4058396a, 0x40584ee1, 0x4058925a,
+    0x40592f6d, 0x405a9f0a, 0x40619618, 0x405fb77c, 0x405d3028, 0x405bef11,
+    0x405a189d, 0x4057aad5, 0x4054f9cc, 0x40518e8c, 0x404fffdd, 0x40510d0d,
+    0x4052400d, 0x4052bab0, 0x40526416, 0x40515cb8, 0x40506aee, 0x404fef8d,
+    0x404ff527, 0x40505b95, 0x4050d61c, 0x4051d0a5, 0x4052abd2, 0x4052f14b,
+    0x4053d196, 0x4054800d, 0x405545f2, 0x4055d71f, 0x40567588, 0x4056de4d,
+    0x40579b72, 0x40584d35, 0x4058cd2f, 0x40594731, 0x4059a53f, 0x405a00ed,
+    0x405a34c1, 0x405a406e, 0x405a1748, 0x405a0300, 0x405a1547, 0x405a66a7,
+    0x405a9be4, 0x405b04b2, 0x405b5754, 0x405b9189, 0x405b9016, 0x405b7a07,
+    0x405b63f9, 0x405b3f45, 0x405b0cb4, 0x405ac80b, 0x405ac1f7, 0x405abbd9,
+    0x405ac86a, 0x405ad72b, 0x405af2f0, 0x405ab465, 0x405a6364, 0x405a1350,
+    0x4059baa3, 0x4059911d, 0x40597921, 0x40595564, 0x40593b8d, 0x4059310f,
+    0x40594e46, 0x40599bae, 0x4059e703, 0x4059feec, 0x405a053a, 0x4059feaa,
+    0x4059d7a0, 0x40599386, 0x40592d0e, 0x4058ce4c, 0x40587335, 0x4058396a,
+    0x40584ee1, 0x4058925a, 0x40592f6d, 0x405a9f0a, 0x40619618, 0x405fb77c,
+    0x405d3028, 0x405bef11, 0x405a189d, 0x4057aad5, 0x4054f9cc, 0x40518e8c,
+    0x404fffdd, 0x40510d0d, 0x4052400d, 0x4052bab0, 0x40526416, 0x40515cb8,
+    0x40506aee, 0x404fef8d, 0x404ff527, 0x40505b95, 0x4050d61c, 0x4051d0a5,
+    0x4052abd2, 0x4052f14b, 0x4053d196, 0x4054800d, 0x405545f2, 0x4055d71f,
+    0x40567588, 0x4056de4d, 0x40579b72, 0x40584d35, 0x4058cd2f, 0x40594731,
+    0x4059a53f, 0x405a00ed, 0x405a34c1, 0x405a406e, 0x405a1748, 0x405a0300,
+    0x405a1547, 0x405a66a7, 0x405a9be4, 0x405b04b2, 0x405b5754, 0x405b9189,
+    0x405b9016, 0x405b7a07, 0x405b63f9, 0x405b3f45, 0x405b0cb4, 0x405ac80b,
+    0x405ac1f7, 0x405abbd9, 0x405ac86a, 0x405ad72b, 0x405af2f0, 0x405ab465,
+    0x405a6364, 0x405a1350, 0x4059baa3, 0x4059911d, 0x40597921, 0x40595564,
+    0x40593b8d, 0x4059310f, 0x40594e46, 0x40599bae, 0x4059e703, 0x4059feec,
+    0x405a053a, 0x4059feaa, 0x4059d7a0, 0x40599386, 0x40592d0e, 0x4058ce4c,
+    0x40587335, 0x4058396a, 0x40584ee1, 0x4058925a, 0x40592f6d, 0x405a9f0a,
+    0x40619618, 0x405fb77c, 0x405d3028, 0x405bef11, 0x405a189d, 0x4057aad5,
+    0x4054f9cc, 0x40518e8c, 0x404fffdd, 0x40510d0d, 0x4052400d, 0x4052bab0,
+    0x40526416, 0x40515cb8, 0x40506aee, 0x404fef8d, 0x404ff527, 0x40505b95,
+    0x4050d61c, 0x4051d0a5, 0x4052abd2, 0x4052f14b, 0x4053d196, 0x4054800d,
+    0x405545f2, 0x4055d71f, 0x40567588, 0x4056de4d, 0x40579b72, 0x40584d35,
+    0x4058cd2f, 0x40594731, 0x4059a53f, 0x405a00ed, 0x405a34c1, 0x405a406e,
+    0x405a1748, 0x405a0300, 0x405a1547, 0x405a66a7, 0x405a9be4, 0x405b04b2,
+    0x405b5754, 0x405b9189, 0x405b9016, 0x405b7a07, 0x405b63f9, 0x405b3f45,
+    0x405b0cb4, 0x405ac80b, 0x405ac1f7, 0x405abbd9, 0x405ac86a, 0x405ad72b,
+    0x405af2f0, 0x405ab465, 0x405a6364, 0x405a1350, 0x4059baa3, 0x4059911d,
+    0x40597921, 0x40595564, 0x40593b8d, 0x4059310f, 0x40594e46, 0x40599bae,
+    0x4059e703, 0x4059feec, 0x405a053a, 0x4059feaa, 0x4059d7a0, 0x40599386,
+    0x40592d0e, 0x4058ce4c, 0x40587335, 0x4058396a, 0x40584ee1, 0x4058925a,
+    0x40592f6d, 0x405a9f0a
+
+};
+
+const int pos_enc_coe_hex[] = {
+    0x3f800000, 0x3f84b063, 0x3f898cc0, 0x3f8e96b2, 0x3f93cfe5, 0x3f993a15,
+    0x3f9ed70c, 0x3fa4a8a8, 0x3faab0d5, 0x3fb0f193, 0x3fb76cf5, 0x3fbe2520,
+    0x3fc51c50, 0x3fcc54d2, 0x3fd3d10c, 0x3fdb9378, 0x3fe39ea9, 0x3febf549,
+    0x3ff49a1b, 0x3ffd8ffe, 0x40036cf4, 0x40083d78, 0x400d3b22, 0x40126799,
+    0x4017c496, 0x401d53df, 0x4023174b, 0x402910c4, 0x402f4244, 0x4035adda,
+    0x403c55a4, 0x40433bd9, 0x404a62c2, 0x4051ccbd, 0x40597c3f, 0x406173d4,
+    0x4069b621, 0x407245e2, 0x407b25ed, 0x40822c9a, 0x4086f161, 0x408be2e0,
+    0x409102bc, 0x409652a6, 0x409bd461, 0x40a189c1, 0x40a774aa, 0x40ad9711,
+    0x40b3f300, 0x40ba8a92, 0x40c15ff6, 0x40c8756f, 0x40cfcd58, 0x40d76a1e,
+    0x40df4e48, 0x40e77c73, 0x40eff755, 0x40f8c1be, 0x4100ef4c, 0x4105a873,
+    0x410a8de6, 0x410fa144, 0x4114e43b, 0x411a588a, 0x41200000, 0x4125dc7c,
+    0x412beff0, 0x41323c5f, 0x4138c3df, 0x413f889a, 0x41468cd0, 0x414dd2d2,
+    0x41555d0a, 0x415d2df7, 0x41654832, 0x416dae69, 0x41766364, 0x417f6a07,
+    0x418462a7, 0x41893c2b, 0x418e432a, 0x4193794e, 0x4198e051, 0x419e79ff,
+    0x41a44831, 0x41aa4cd6, 0x41b089ea, 0x41b70180, 0x41bdb5bc, 0x41c4a8d7,
+    0x41cbdd1e, 0x41d354f5, 0x41db12d6, 0x41e31950, 0x41eb6b0d, 0x41f40ad0,
+    0x41fcfb72, 0x42031ff6, 0x4207eda7, 0x420ce865, 0x421211d5, 0x42176bad,
+    0x421cf7b4, 0x4222b7c0, 0x4228adb9, 0x422edb98, 0x4235436b, 0x423be74f,
+    0x4242c979, 0x4249ec31, 0x425151d4, 0x4258fcd6, 0x4260efc0, 0x42692d37,
+    0x4271b7f3, 0x427a92cb, 0x4281e057, 0x4286a253, 0x428b90ed, 0x4290adc8,
+    0x4295fa95, 0x429b7917, 0x42a12b1f, 0x42a71290, 0x42ad3160, 0x42b38995,
+    0x42ba1d4a, 0x42c0eead, 0x42c80000, 0x42cf539b, 0x42d6ebec, 0x42decb76,
+    0x42e6f4d6, 0x42ef6ac1, 0x42f83003, 0x4300a3c3, 0x43055a26, 0x430a3cbb,
+    0x430f4d1f, 0x43148d01, 0x4319fe1e, 0x431fa244, 0x43257b51, 0x432b8b36,
+    0x4331d3f4, 0x433857a1, 0x433f1865, 0x4346187e, 0x434d5a3e, 0x4354e00b,
+    0x435cac64, 0x4364c1e0, 0x436d232b, 0x4375d30c, 0x437ed466, 0x43841519,
+    0x4388ebc5, 0x438defd2, 0x439322e8, 0x439886c2, 0x439e1d27, 0x43a3e7f3,
+    0x43a9e911, 0x43b0227e, 0x43b6964a, 0x43bd4698, 0x43c435a1, 0x43cb65b0,
+    0x43d2d927, 0x43da927e, 0x43e29445, 0x43eae123, 0x43f37bd8, 0x43fc673e,
+    0x4402d325, 0x44079e06, 0x440c95d8, 0x4411bc42, 0x441712f8, 0x441c9bbf,
+    0x4422586d, 0x44284ae8, 0x442e7528, 0x4434d93a, 0x443b793b, 0x4442575d,
+    0x444975e6, 0x4450d734, 0x44587db7, 0x44606bfa, 0x4468a49c, 0x44712a58,
+    0x447a0000, 0x44819441, 0x44865373, 0x448b3f2a, 0x44905906, 0x4495a2b9,
+    0x449b1e02, 0x44a0ccb4, 0x44a6b0b0, 0x44accbe9, 0x44b32067, 0x44b9b042,
+    0x44c07da6, 0x44c78ad5, 0x44ceda26, 0x44d66e03, 0x44de48f1, 0x44e66d89,
+    0x44eede7f, 0x44f79e9e, 0x45005867, 0x45050c07, 0x4509ebbf, 0x450ef92c,
+    0x451435fb, 0x4519a3e8, 0x451f44bf, 0x45251a60, 0x452b26b7, 0x45316bc7,
+    0x4537eba3, 0x453ea872, 0x4545a471, 0x454ce1f0, 0x45546355, 0x455c2b1d,
+    0x45643bdc, 0x456c983e, 0x45754309, 0x457e3f1c, 0x4583c7b8, 0x45889b8f,
+    0x458d9cab, 0x4592ccb6, 0x45982d67, 0x459dc087, 0x45a387ee, 0x45a98587,
+    0x45afbb4e, 0x45b62b53, 0x45bcd7b6, 0x45c3c2af, 0x45caee88, 0x45d25da1,
+    0x45da1272, 0x45e20f88, 0x45ea5789, 0x45f2ed34, 0x45fbd360, 0x46028680,
+    0x46074e93, 0x460c437c, 0x461166e2, 0x4616ba77};
+
+const int pos_enc_div_term_hex[] = {
+    0x3f800000, 0x3f76f410, 0x3f6e39f8, 0x3f65ced3, 0x3f5dafd7, 0x3f55da52,
+    0x3f4e4bac, 0x3f470165, 0x3f3ff911, 0x3f39305c, 0x3f32a506, 0x3f2c54e5,
+    0x3f263de0, 0x3f205df3, 0x3f1ab32b, 0x3f153ba8, 0x3f0ff59a, 0x3f0adf41,
+    0x3f05f6ee, 0x3f013b01, 0x3ef953cf, 0x3ef0843c, 0x3ee80460, 0x3edfd167,
+    0x3ed7e89b, 0x3ed0475c, 0x3ec8eb24, 0x3ec1d181, 0x3ebaf81a, 0x3eb45caa,
+    0x3eadfcff, 0x3ea7d6fd, 0x3ea1e89b, 0x3e9c2fe1, 0x3e96aaea, 0x3e9157e1,
+    0x3e8c3504, 0x3e87409d, 0x3e827909, 0x3e7bb965, 0x3e72d424, 0x3e6a3f5c,
+    0x3e61f836, 0x3e59fbf3, 0x3e5247ed, 0x3e4ad998, 0x3e43ae7c, 0x3e3cc43a,
+    0x3e361887, 0x3e2fa92d, 0x3e29740a, 0x3e23770f, 0x3e1db040, 0x3e181db4,
+    0x3e12bd91, 0x3e0d8e0f, 0x3e088d77, 0x3e03ba20, 0x3dfe24e1, 0x3df529bb,
+    0x3dec7fd5, 0x3de42450, 0x3ddc1466, 0x3dd44d6c, 0x3dcccccd, 0x3dc5900d,
+    0x3dbe94c7, 0x3db7d8a9, 0x3db15978, 0x3dab150e, 0x3da50957, 0x3d9f3451,
+    0x3d99940e, 0x3d9426b0, 0x3d8eea6c, 0x3d89dd84, 0x3d84fe4d, 0x3d804b29,
+    0x3d778512, 0x3d6ec5da, 0x3d6655c3, 0x3d5e3202, 0x3d5657e4, 0x3d4ec4ce,
+    0x3d47763f, 0x3d4069ca, 0x3d399d19, 0x3d330dec, 0x3d2cba15, 0x3d269f7d,
+    0x3d20bc1d, 0x3d1b0e01, 0x3d159348, 0x3d104a21, 0x3d0b30cc, 0x3d064597,
+    0x3d0186e2, 0x3cf9e635, 0x3cf11176, 0x3ce88c9c, 0x3ce054d2, 0x3cd86761,
+    0x3cd0c1a8, 0x3cc9611d, 0x3cc24350, 0x3cbb65e3, 0x3cb4c691, 0x3cae6328,
+    0x3ca8398b, 0x3ca247ad, 0x3c9c8b97, 0x3c970362, 0x3c91ad39, 0x3c8c8757,
+    0x3c879008, 0x3c82c5a5, 0x3c7c4d33, 0x3c7362b9, 0x3c6ac8e7, 0x3c627ce5,
+    0x3c5a7bf1, 0x3c52c366, 0x3c4b50b4, 0x3c442163, 0x3c3d3311, 0x3c368373,
+    0x3c301052, 0x3c29d789, 0x3c23d70a, 0x3c1e0cd7, 0x3c187705, 0x3c1313ba,
+    0x3c0de12d, 0x3c08dda5, 0x3c040779, 0x3bfeba1b, 0x3bf5b9b0, 0x3bed0ab3,
+    0x3be4aa46, 0x3bdc95a0, 0x3bd4ca14, 0x3bcd450e, 0x3bc6040e, 0x3bbf04ae,
+    0x3bb8449c, 0x3bb1c19b, 0x3bab7983, 0x3ba56a3f, 0x3b9f91cc, 0x3b99ee3b,
+    0x3b947dae, 0x3b8f3e56, 0x3b8a2e77, 0x3b854c64, 0x3b80967d, 0x3b781668,
+    0x3b6f520d, 0x3b66dd02, 0x3b5eb47a, 0x3b56d5bf, 0x3b4f3e37, 0x3b47eb5e,
+    0x3b40dac5, 0x3b3a0a16, 0x3b33770f, 0x3b2d1f81, 0x3b270153, 0x3b211a7e,
+    0x3b1b690d, 0x3b15eb1c, 0x3b109edb, 0x3b0b8287, 0x3b06946f, 0x3b01d2f1,
+    0x3afa78f1, 0x3af19f03, 0x3ae91528, 0x3ae0d88b, 0x3ad8e673, 0x3ad13c3c,
+    0x3ac9d75c, 0x3ac2b561, 0x3abbd3ec, 0x3ab530b7, 0x3aaec98e, 0x3aa89c52,
+    0x3aa2a6f6, 0x3a9ce782, 0x3a975c0e, 0x3a9202c3, 0x3a8cd9db, 0x3a87dfa1,
+    0x3a83126f, 0x3a7ce158, 0x3a73f1a2, 0x3a6b52c4, 0x3a6301e2, 0x3a5afc3b,
+    0x3a533f27, 0x3a4bc816, 0x3a44948c, 0x3a3da229, 0x3a36ee9e, 0x3a3077b3,
+    0x3a2a3b43, 0x3a24373e, 0x3a1e69a5, 0x3a18d08b, 0x3a136a16, 0x3a0e347c,
+    0x3a092e02, 0x3a0454ff, 0x39ff4fad, 0x39f649f8, 0x39ed95e3, 0x39e5308a,
+    0x39dd1726, 0x39d54706, 0x39cdbd95, 0x39c67853, 0x39bf74d7, 0x39b8b0cf,
+    0x39b229fb, 0x39abde33, 0x39a5cb5f, 0x399fef7e, 0x399a489e, 0x3994d4df,
+    0x398f9272, 0x398a7f9b, 0x39859aa9, 0x3980e1fe, 0x3978a814, 0x396fde93,
+    0x39676491, 0x395f373e, 0x395753e5, 0x394fb7e7, 0x394860c1, 0x39414c02,
+    0x393a7753, 0x3933e06f, 0x392d8529, 0x39276363, 0x39217917, 0x391bc44d,
+    0x39164323, 0x3910f3c6, 0x390bd472, 0x3906e374, 0x39021f2b, 0x38fb0c03,
+    0x38f22ce3, 0x38e99e04, 0x38e15c92, 0x38d965ce};
+#endif
+
+} // namespace funasr

runtime/onnxruntime/src/punc-model.cpp

@@ -0,0 +1,30 @@
+#include "precomp.h"
+
+namespace funasr {
+PuncModel *CreatePuncModel(std::map<std::string, std::string>& model_path, int thread_num, PUNC_TYPE type)
+{
+    PuncModel *mm;
+    if (type==PUNC_OFFLINE){
+        mm = new CTTransformer();
+    }else if(type==PUNC_ONLINE){
+        mm = new CTTransformerOnline();
+    }else{
+        LOG(ERROR) << "Wrong PUNC TYPE";
+        exit(-1);
+    }
+    string punc_model_path;
+    string punc_config_path;
+    string token_file;
+
+    punc_model_path = PathAppend(model_path.at(MODEL_DIR), MODEL_NAME);
+    if(model_path.find(QUANTIZE) != model_path.end() && model_path.at(QUANTIZE) == "true"){
+        punc_model_path = PathAppend(model_path.at(MODEL_DIR), QUANT_MODEL_NAME);
+    }
+    punc_config_path = PathAppend(model_path.at(MODEL_DIR), PUNC_CONFIG_NAME);
+    token_file = PathAppend(model_path.at(MODEL_DIR), TOKEN_PATH);
+
+    mm->InitPunc(punc_model_path, punc_config_path, token_file, thread_num);
+    return mm;
+}
+
+} // namespace funasr

runtime/onnxruntime/src/resample.cpp

@@ -0,0 +1,307 @@
+/**
+ * Copyright     2013  Pegah Ghahremani
+ *               2014  IMSL, PKU-HKUST (author: Wei Shi)
+ *               2014  Yanqing Sun, Junjie Wang
+ *               2014  Johns Hopkins University (author: Daniel Povey)
+ * Copyright     2023  Xiaomi Corporation (authors: Fangjun Kuang)
+ *
+ * See LICENSE for clarification regarding multiple authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// this file is copied and modified from
+// kaldi/src/feat/resample.cc
+
+#include "resample.h"
+
+#include <assert.h>
+#include <math.h>
+#include <stdio.h>
+
+#include <cstdlib>
+#include <type_traits>
+
+namespace funasr {
+#ifndef M_2PI
+#define M_2PI 6.283185307179586476925286766559005
+#endif
+
+#ifndef M_PI
+#define M_PI 3.1415926535897932384626433832795
+#endif
+
+template <class I>
+I Gcd(I m, I n) {
+  // this function is copied from kaldi/src/base/kaldi-math.h
+  if (m == 0 || n == 0) {
+    if (m == 0 && n == 0) {  // gcd not defined, as all integers are divisors.
+      fprintf(stderr, "Undefined GCD since m = 0, n = 0.\n");
+      exit(-1);
+    }
+    return (m == 0 ? (n > 0 ? n : -n) : (m > 0 ? m : -m));
+    // return absolute value of whichever is nonzero
+  }
+  // could use compile-time assertion
+  // but involves messing with complex template stuff.
+  static_assert(std::is_integral<I>::value, "");
+  while (1) {
+    m %= n;
+    if (m == 0) return (n > 0 ? n : -n);
+    n %= m;
+    if (n == 0) return (m > 0 ? m : -m);
+  }
+}
+
+/// Returns the least common multiple of two integers.  Will
+/// crash unless the inputs are positive.
+template <class I>
+I Lcm(I m, I n) {
+  // This function is copied from kaldi/src/base/kaldi-math.h
+  assert(m > 0 && n > 0);
+  I gcd = Gcd(m, n);
+  return gcd * (m / gcd) * (n / gcd);
+}
+
+static float DotProduct(const float *a, const float *b, int32_t n) {
+  float sum = 0;
+  for (int32_t i = 0; i != n; ++i) {
+    sum += a[i] * b[i];
+  }
+  return sum;
+}
+
+LinearResample::LinearResample(int32_t samp_rate_in_hz,
+                               int32_t samp_rate_out_hz, float filter_cutoff_hz,
+                               int32_t num_zeros)
+    : samp_rate_in_(samp_rate_in_hz),
+      samp_rate_out_(samp_rate_out_hz),
+      filter_cutoff_(filter_cutoff_hz),
+      num_zeros_(num_zeros) {
+  assert(samp_rate_in_hz > 0.0 && samp_rate_out_hz > 0.0 &&
+         filter_cutoff_hz > 0.0 && filter_cutoff_hz * 2 <= samp_rate_in_hz &&
+         filter_cutoff_hz * 2 <= samp_rate_out_hz && num_zeros > 0);
+
+  // base_freq is the frequency of the repeating unit, which is the gcd
+  // of the input frequencies.
+  int32_t base_freq = Gcd(samp_rate_in_, samp_rate_out_);
+  input_samples_in_unit_ = samp_rate_in_ / base_freq;
+  output_samples_in_unit_ = samp_rate_out_ / base_freq;
+
+  SetIndexesAndWeights();
+  Reset();
+}
+
+void LinearResample::SetIndexesAndWeights() {
+  first_index_.resize(output_samples_in_unit_);
+  weights_.resize(output_samples_in_unit_);
+
+  double window_width = num_zeros_ / (2.0 * filter_cutoff_);
+
+  for (int32_t i = 0; i < output_samples_in_unit_; i++) {
+    double output_t = i / static_cast<double>(samp_rate_out_);
+    double min_t = output_t - window_width, max_t = output_t + window_width;
+    // we do ceil on the min and floor on the max, because if we did it
+    // the other way around we would unnecessarily include indexes just
+    // outside the window, with zero coefficients.  It's possible
+    // if the arguments to the ceil and floor expressions are integers
+    // (e.g. if filter_cutoff_ has an exact ratio with the sample rates),
+    // that we unnecessarily include something with a zero coefficient,
+    // but this is only a slight efficiency issue.
+    int32_t min_input_index = ceil(min_t * samp_rate_in_),
+            max_input_index = floor(max_t * samp_rate_in_),
+            num_indices = max_input_index - min_input_index + 1;
+    first_index_[i] = min_input_index;
+    weights_[i].resize(num_indices);
+    for (int32_t j = 0; j < num_indices; j++) {
+      int32_t input_index = min_input_index + j;
+      double input_t = input_index / static_cast<double>(samp_rate_in_),
+             delta_t = input_t - output_t;
+      // sign of delta_t doesn't matter.
+      weights_[i][j] = FilterFunc(delta_t) / samp_rate_in_;
+    }
+  }
+}
+
+/** Here, t is a time in seconds representing an offset from
+    the center of the windowed filter function, and FilterFunction(t)
+    returns the windowed filter function, described
+    in the header as h(t) = f(t)g(t), evaluated at t.
+*/
+float LinearResample::FilterFunc(float t) const {
+  float window,  // raised-cosine (Hanning) window of width
+                 // num_zeros_/2*filter_cutoff_
+      filter;    // sinc filter function
+  if (fabs(t) < num_zeros_ / (2.0 * filter_cutoff_))
+    window = 0.5 * (1 + cos(M_2PI * filter_cutoff_ / num_zeros_ * t));
+  else
+    window = 0.0;  // outside support of window function
+  if (t != 0)
+    filter = sin(M_2PI * filter_cutoff_ * t) / (M_PI * t);
+  else
+    filter = 2 * filter_cutoff_;  // limit of the function at t = 0
+  return filter * window;
+}
+
+void LinearResample::Reset() {
+  input_sample_offset_ = 0;
+  output_sample_offset_ = 0;
+  input_remainder_.resize(0);
+}
+
+void LinearResample::Resample(const float *input, int32_t input_dim, bool flush,
+                              std::vector<float> *output) {
+  int64_t tot_input_samp = input_sample_offset_ + input_dim,
+          tot_output_samp = GetNumOutputSamples(tot_input_samp, flush);
+
+  assert(tot_output_samp >= output_sample_offset_);
+
+  output->resize(tot_output_samp - output_sample_offset_);
+
+  // samp_out is the index into the total output signal, not just the part
+  // of it we are producing here.
+  for (int64_t samp_out = output_sample_offset_; samp_out < tot_output_samp;
+       samp_out++) {
+    int64_t first_samp_in;
+    int32_t samp_out_wrapped;
+    GetIndexes(samp_out, &first_samp_in, &samp_out_wrapped);
+    const std::vector<float> &weights = weights_[samp_out_wrapped];
+    // first_input_index is the first index into "input" that we have a weight
+    // for.
+    int32_t first_input_index =
+        static_cast<int32_t>(first_samp_in - input_sample_offset_);
+    float this_output;
+    if (first_input_index >= 0 &&
+        first_input_index + static_cast<int32_t>(weights.size()) <= input_dim) {
+      this_output =
+          DotProduct(input + first_input_index, weights.data(), weights.size());
+    } else {  // Handle edge cases.
+      this_output = 0.0;
+      for (int32_t i = 0; i < static_cast<int32_t>(weights.size()); i++) {
+        float weight = weights[i];
+        int32_t input_index = first_input_index + i;
+        if (input_index < 0 &&
+            static_cast<int32_t>(input_remainder_.size()) + input_index >= 0) {
+          this_output +=
+              weight * input_remainder_[input_remainder_.size() + input_index];
+        } else if (input_index >= 0 && input_index < input_dim) {
+          this_output += weight * input[input_index];
+        } else if (input_index >= input_dim) {
+          // We're past the end of the input and are adding zero; should only
+          // happen if the user specified flush == true, or else we would not
+          // be trying to output this sample.
+          assert(flush);
+        }
+      }
+    }
+    int32_t output_index =
+        static_cast<int32_t>(samp_out - output_sample_offset_);
+    (*output)[output_index] = this_output;
+  }
+
+  if (flush) {
+    Reset();  // Reset the internal state.
+  } else {
+    SetRemainder(input, input_dim);
+    input_sample_offset_ = tot_input_samp;
+    output_sample_offset_ = tot_output_samp;
+  }
+}
+
+int64_t LinearResample::GetNumOutputSamples(int64_t input_num_samp,
+                                            bool flush) const {
+  // For exact computation, we measure time in "ticks" of 1.0 / tick_freq,
+  // where tick_freq is the least common multiple of samp_rate_in_ and
+  // samp_rate_out_.
+  int32_t tick_freq = Lcm(samp_rate_in_, samp_rate_out_);
+  int32_t ticks_per_input_period = tick_freq / samp_rate_in_;
+
+  // work out the number of ticks in the time interval
+  // [ 0, input_num_samp/samp_rate_in_ ).
+  int64_t interval_length_in_ticks = input_num_samp * ticks_per_input_period;
+  if (!flush) {
+    float window_width = num_zeros_ / (2.0 * filter_cutoff_);
+    // To count the window-width in ticks we take the floor.  This
+    // is because since we're looking for the largest integer num-out-samp
+    // that fits in the interval, which is open on the right, a reduction
+    // in interval length of less than a tick will never make a difference.
+    // For example, the largest integer in the interval [ 0, 2 ) and the
+    // largest integer in the interval [ 0, 2 - 0.9 ) are the same (both one).
+    // So when we're subtracting the window-width we can ignore the fractional
+    // part.
+    int32_t window_width_ticks = floor(window_width * tick_freq);
+    // The time-period of the output that we can sample gets reduced
+    // by the window-width (which is actually the distance from the
+    // center to the edge of the windowing function) if we're not
+    // "flushing the output".
+    interval_length_in_ticks -= window_width_ticks;
+  }
+  if (interval_length_in_ticks <= 0) return 0;
+
+  int32_t ticks_per_output_period = tick_freq / samp_rate_out_;
+  // Get the last output-sample in the closed interval, i.e. replacing [ ) with
+  // [ ].  Note: integer division rounds down.  See
+  // http://en.wikipedia.org/wiki/Interval_(mathematics) for an explanation of
+  // the notation.
+  int64_t last_output_samp = interval_length_in_ticks / ticks_per_output_period;
+  // We need the last output-sample in the open interval, so if it takes us to
+  // the end of the interval exactly, subtract one.
+  if (last_output_samp * ticks_per_output_period == interval_length_in_ticks)
+    last_output_samp--;
+
+  // First output-sample index is zero, so the number of output samples
+  // is the last output-sample plus one.
+  int64_t num_output_samp = last_output_samp + 1;
+  return num_output_samp;
+}
+
+// inline
+void LinearResample::GetIndexes(int64_t samp_out, int64_t *first_samp_in,
+                                int32_t *samp_out_wrapped) const {
+  // A unit is the smallest nonzero amount of time that is an exact
+  // multiple of the input and output sample periods.  The unit index
+  // is the answer to "which numbered unit we are in".
+  int64_t unit_index = samp_out / output_samples_in_unit_;
+  // samp_out_wrapped is equal to samp_out % output_samples_in_unit_
+  *samp_out_wrapped =
+      static_cast<int32_t>(samp_out - unit_index * output_samples_in_unit_);
+  *first_samp_in =
+      first_index_[*samp_out_wrapped] + unit_index * input_samples_in_unit_;
+}
+
+void LinearResample::SetRemainder(const float *input, int32_t input_dim) {
+  std::vector<float> old_remainder(input_remainder_);
+  // max_remainder_needed is the width of the filter from side to side,
+  // measured in input samples.  you might think it should be half that,
+  // but you have to consider that you might be wanting to output samples
+  // that are "in the past" relative to the beginning of the latest
+  // input... anyway, storing more remainder than needed is not harmful.
+  int32_t max_remainder_needed =
+      ceil(samp_rate_in_ * num_zeros_ / filter_cutoff_);
+  input_remainder_.resize(max_remainder_needed);
+  for (int32_t index = -static_cast<int32_t>(input_remainder_.size());
+       index < 0; index++) {
+    // we interpret "index" as an offset from the end of "input" and
+    // from the end of input_remainder_.
+    int32_t input_index = index + input_dim;
+    if (input_index >= 0) {
+      input_remainder_[index + static_cast<int32_t>(input_remainder_.size())] =
+          input[input_index];
+    } else if (input_index + static_cast<int32_t>(old_remainder.size()) >= 0) {
+      input_remainder_[index + static_cast<int32_t>(input_remainder_.size())] =
+          old_remainder[input_index +
+                        static_cast<int32_t>(old_remainder.size())];
+      // else leave it at zero.
+    }
+  }
+}
+} // namespace funasr

runtime/onnxruntime/src/resample.h

@@ -0,0 +1,138 @@
+/**
+ * Copyright     2013  Pegah Ghahremani
+ *               2014  IMSL, PKU-HKUST (author: Wei Shi)
+ *               2014  Yanqing Sun, Junjie Wang
+ *               2014  Johns Hopkins University (author: Daniel Povey)
+ * Copyright     2023  Xiaomi Corporation (authors: Fangjun Kuang)
+ *
+ * See LICENSE for clarification regarding multiple authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// this file is copied and modified from
+// kaldi/src/feat/resample.h
+#pragma once 
+#include <cstdint>
+#include <vector>
+
+namespace funasr {
+/*
+   We require that the input and output sampling rate be specified as
+   integers, as this is an easy way to specify that their ratio be rational.
+*/
+
+class LinearResample {
+ public:
+  /// Constructor.  We make the input and output sample rates integers, because
+  /// we are going to need to find a common divisor.  This should just remind
+  /// you that they need to be integers.  The filter cutoff needs to be less
+  /// than samp_rate_in_hz/2 and less than samp_rate_out_hz/2.  num_zeros
+  /// controls the sharpness of the filter, more == sharper but less efficient.
+  /// We suggest around 4 to 10 for normal use.
+  LinearResample(int32_t samp_rate_in_hz, int32_t samp_rate_out_hz,
+                 float filter_cutoff_hz, int32_t num_zeros);
+
+  /// Calling the function Reset() resets the state of the object prior to
+  /// processing a new signal; it is only necessary if you have called
+  /// Resample(x, x_size, false, y) for some signal, leading to a remainder of
+  /// the signal being called, but then abandon processing the signal before
+  /// calling Resample(x, x_size, true, y) for the last piece.  Call it
+  /// unnecessarily between signals will not do any harm.
+  void Reset();
+
+  /// This function does the resampling.  If you call it with flush == true and
+  /// you have never called it with flush == false, it just resamples the input
+  /// signal (it resizes the output to a suitable number of samples).
+  ///
+  /// You can also use this function to process a signal a piece at a time.
+  /// suppose you break it into piece1, piece2, ... pieceN.  You can call
+  /// \code{.cc}
+  /// Resample(piece1, piece1_size, false, &output1);
+  /// Resample(piece2, piece2_size, false, &output2);
+  /// Resample(piece3, piece3_size, true, &output3);
+  /// \endcode
+  /// If you call it with flush == false, it won't output the last few samples
+  /// but will remember them, so that if you later give it a second piece of
+  /// the input signal it can process it correctly.
+  /// If your most recent call to the object was with flush == false, it will
+  /// have internal state; you can remove this by calling Reset().
+  /// Empty input is acceptable.
+  void Resample(const float *input, int32_t input_dim, bool flush,
+                std::vector<float> *output);
+
+  //// Return the input and output sampling rates (for checks, for example)
+  int32_t GetInputSamplingRate() const { return samp_rate_in_; }
+  int32_t GetOutputSamplingRate() const { return samp_rate_out_; }
+
+ private:
+  void SetIndexesAndWeights();
+
+  float FilterFunc(float) const;
+
+  /// This function outputs the number of output samples we will output
+  /// for a signal with "input_num_samp" input samples.  If flush == true,
+  /// we return the largest n such that
+  /// (n/samp_rate_out_) is in the interval [ 0, input_num_samp/samp_rate_in_ ),
+  /// and note that the interval is half-open.  If flush == false,
+  /// define window_width as num_zeros / (2.0 * filter_cutoff_);
+  /// we return the largest n such that (n/samp_rate_out_) is in the interval
+  /// [ 0, input_num_samp/samp_rate_in_ - window_width ).
+  int64_t GetNumOutputSamples(int64_t input_num_samp, bool flush) const;
+
+  /// Given an output-sample index, this function outputs to *first_samp_in the
+  /// first input-sample index that we have a weight on (may be negative),
+  /// and to *samp_out_wrapped the index into weights_ where we can get the
+  /// corresponding weights on the input.
+  inline void GetIndexes(int64_t samp_out, int64_t *first_samp_in,
+                         int32_t *samp_out_wrapped) const;
+
+  void SetRemainder(const float *input, int32_t input_dim);
+
+ private:
+  // The following variables are provided by the user.
+  int32_t samp_rate_in_;
+  int32_t samp_rate_out_;
+  float filter_cutoff_;
+  int32_t num_zeros_;
+
+  int32_t input_samples_in_unit_;  ///< The number of input samples in the
+                                   ///< smallest repeating unit: num_samp_in_ =
+                                   ///< samp_rate_in_hz / Gcd(samp_rate_in_hz,
+                                   ///< samp_rate_out_hz)
+
+  int32_t output_samples_in_unit_;  ///< The number of output samples in the
+                                    ///< smallest repeating unit: num_samp_out_
+                                    ///< = samp_rate_out_hz /
+                                    ///< Gcd(samp_rate_in_hz, samp_rate_out_hz)
+
+  /// The first input-sample index that we sum over, for this output-sample
+  /// index.  May be negative; any truncation at the beginning is handled
+  /// separately.  This is just for the first few output samples, but we can
+  /// extrapolate the correct input-sample index for arbitrary output samples.
+  std::vector<int32_t> first_index_;
+
+  /// Weights on the input samples, for this output-sample index.
+  std::vector<std::vector<float>> weights_;
+
+  // the following variables keep track of where we are in a particular signal,
+  // if it is being provided over multiple calls to Resample().
+
+  int64_t input_sample_offset_;   ///< The number of input samples we have
+                                  ///< already received for this signal
+                                  ///< (including anything in remainder_)
+  int64_t output_sample_offset_;  ///< The number of samples we have already
+                                  ///< output for this signal.
+  std::vector<float> input_remainder_;  ///< A small trailing part of the
+                                        ///< previously seen input signal.
+};
+} // namespace funasr

runtime/onnxruntime/src/seg_dict.cpp

@@ -0,0 +1,54 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+#include "precomp.h"
+//#include "util.h"
+//#include "seg_dict.h"
+#include <glog/logging.h>
+
+#include <fstream>
+#include <iostream>
+#include <list>
+#include <sstream>
+#include <string>
+
+using namespace std;
+
+namespace funasr {
+SegDict::SegDict(const char *filename)
+{
+    ifstream in(filename);
+    if (!in) {
+      LOG(ERROR) << filename << " open failed !!";
+      return;
+    }
+    string textline;
+    while (getline(in, textline)) {
+      std::vector<string> line_item = split(textline, '\t');
+      //std::cout << textline << std::endl;
+      if (line_item.size() > 1) {
+        std::string word = line_item[0];
+        std::string segs = line_item[1];
+        std::vector<string> segs_vec = split(segs, ' ');
+        seg_dict[word] = segs_vec;
+      }
+    }
+    LOG(INFO) << "load seg dict successfully";
+}
+std::vector<std::string> SegDict::GetTokensByWord(const std::string &word) {
+  if (seg_dict.count(word))
+    return seg_dict[word];
+  else {
+    LOG(INFO)<< word <<" is OOV!";
+    std::vector<string> vec;
+    return vec;
+  }
+}
+
+SegDict::~SegDict()
+{
+}
+
+
+} // namespace funasr

runtime/onnxruntime/src/seg_dict.h

@@ -0,0 +1,26 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+#ifndef SEG_DICT_H
+#define SEG_DICT_H
+
+#include <stdint.h>
+#include <string>
+#include <vector>
+#include <map>
+using namespace std;
+
+namespace funasr {
+class SegDict {
+  private:
+    std::map<string, std::vector<string>> seg_dict;
+
+  public:
+    SegDict(const char *filename);
+    ~SegDict();
+    std::vector<std::string> GetTokensByWord(const std::string &word);
+};
+
+} // namespace funasr
+#endif

runtime/onnxruntime/src/sensevoice-small.cpp

@@ -0,0 +1,525 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#include "precomp.h"
+#include "sensevoice-small.h"
+#include <cstddef>
+
+using namespace std;
+namespace funasr {
+
+SenseVoiceSmall::SenseVoiceSmall()
+:use_hotword(false),
+ env_(ORT_LOGGING_LEVEL_ERROR, "sensevoice"),session_options_{} {
+}
+
+// offline
+void SenseVoiceSmall::InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){
+    LoadConfigFromYaml(am_config.c_str());
+    // knf options
+    fbank_opts_.frame_opts.dither = 0;
+    fbank_opts_.mel_opts.num_bins = n_mels;
+    fbank_opts_.frame_opts.samp_freq = asr_sample_rate;
+    fbank_opts_.frame_opts.window_type = window_type;
+    fbank_opts_.frame_opts.frame_shift_ms = frame_shift;
+    fbank_opts_.frame_opts.frame_length_ms = frame_length;
+    fbank_opts_.energy_floor = 0;
+    fbank_opts_.mel_opts.debug_mel = false;
+
+    // session_options_.SetInterOpNumThreads(1);
+    session_options_.SetIntraOpNumThreads(thread_num);
+    session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
+    // DisableCpuMemArena can improve performance
+    session_options_.DisableCpuMemArena();
+
+    try {
+        m_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(am_model).c_str(), session_options_);
+        LOG(INFO) << "Successfully load model from " << am_model;
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load am onnx model: " << e.what();
+        exit(-1);
+    }
+
+    GetInputNames(m_session_.get(), m_strInputNames, m_szInputNames);
+    GetOutputNames(m_session_.get(), m_strOutputNames, m_szOutputNames);
+    vocab = new Vocab(token_file.c_str());
+    LoadCmvn(am_cmvn.c_str());
+}
+
+// online
+void SenseVoiceSmall::InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){
+    
+    LoadOnlineConfigFromYaml(am_config.c_str());
+    // knf options
+    fbank_opts_.frame_opts.dither = 0;
+    fbank_opts_.mel_opts.num_bins = n_mels;
+    fbank_opts_.frame_opts.samp_freq = asr_sample_rate;
+    fbank_opts_.frame_opts.window_type = window_type;
+    fbank_opts_.frame_opts.frame_shift_ms = frame_shift;
+    fbank_opts_.frame_opts.frame_length_ms = frame_length;
+    fbank_opts_.energy_floor = 0;
+    fbank_opts_.mel_opts.debug_mel = false;
+
+    // session_options_.SetInterOpNumThreads(1);
+    session_options_.SetIntraOpNumThreads(thread_num);
+    session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
+    // DisableCpuMemArena can improve performance
+    session_options_.DisableCpuMemArena();
+
+    try {
+        encoder_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(en_model).c_str(), session_options_);
+        LOG(INFO) << "Successfully load model from " << en_model;
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load am encoder model: " << e.what();
+        exit(-1);
+    }
+
+    try {
+        decoder_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(de_model).c_str(), session_options_);
+        LOG(INFO) << "Successfully load model from " << de_model;
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load am decoder model: " << e.what();
+        exit(-1);
+    }
+
+    // encoder
+    string strName;
+    GetInputName(encoder_session_.get(), strName);
+    en_strInputNames.push_back(strName.c_str());
+    GetInputName(encoder_session_.get(), strName,1);
+    en_strInputNames.push_back(strName);
+    
+    GetOutputName(encoder_session_.get(), strName);
+    en_strOutputNames.push_back(strName);
+    GetOutputName(encoder_session_.get(), strName,1);
+    en_strOutputNames.push_back(strName);
+    GetOutputName(encoder_session_.get(), strName,2);
+    en_strOutputNames.push_back(strName);
+
+    for (auto& item : en_strInputNames)
+        en_szInputNames_.push_back(item.c_str());
+    for (auto& item : en_strOutputNames)
+        en_szOutputNames_.push_back(item.c_str());
+
+    // decoder
+    int de_input_len = 4 + fsmn_layers;
+    int de_out_len = 2 + fsmn_layers;
+    for(int i=0;i<de_input_len; i++){
+        GetInputName(decoder_session_.get(), strName, i);
+        de_strInputNames.push_back(strName.c_str());
+    }
+
+    for(int i=0;i<de_out_len; i++){
+        GetOutputName(decoder_session_.get(), strName,i);
+        de_strOutputNames.push_back(strName);
+    }
+
+    for (auto& item : de_strInputNames)
+        de_szInputNames_.push_back(item.c_str());
+    for (auto& item : de_strOutputNames)
+        de_szOutputNames_.push_back(item.c_str());
+
+    online_vocab = new Vocab(token_file.c_str());
+    phone_set_ = new PhoneSet(token_file.c_str());
+    LoadCmvn(am_cmvn.c_str());
+}
+
+// 2pass
+void SenseVoiceSmall::InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, 
+    const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, const std::string &online_token_file, int thread_num){
+    // online
+    InitAsr(en_model, de_model, am_cmvn, am_config, online_token_file, thread_num);
+
+    // offline
+    try {
+        m_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(am_model).c_str(), session_options_);
+        LOG(INFO) << "Successfully load model from " << am_model;
+    } catch (std::exception const &e) {
+        LOG(ERROR) << "Error when load am onnx model: " << e.what();
+        exit(-1);
+    }
+
+    GetInputNames(m_session_.get(), m_strInputNames, m_szInputNames);
+    GetOutputNames(m_session_.get(), m_strOutputNames, m_szOutputNames);
+    vocab = new Vocab(token_file.c_str());
+}
+
+void SenseVoiceSmall::LoadOnlineConfigFromYaml(const char* filename){
+
+    YAML::Node config;
+    try{
+        config = YAML::LoadFile(filename);
+    }catch(exception const &e){
+        LOG(ERROR) << "Error loading file, yaml file error or not exist.";
+        exit(-1);
+    }
+
+    try{
+        YAML::Node frontend_conf = config["frontend_conf"];
+        YAML::Node encoder_conf = config["encoder_conf"];
+        YAML::Node decoder_conf = config["decoder_conf"];
+        YAML::Node predictor_conf = config["predictor_conf"];
+
+        this->window_type = frontend_conf["window"].as<string>();
+        this->n_mels = frontend_conf["n_mels"].as<int>();
+        this->frame_length = frontend_conf["frame_length"].as<int>();
+        this->frame_shift = frontend_conf["frame_shift"].as<int>();
+        this->lfr_m = frontend_conf["lfr_m"].as<int>();
+        this->lfr_n = frontend_conf["lfr_n"].as<int>();
+
+        this->encoder_size = encoder_conf["output_size"].as<int>();
+        this->fsmn_dims = encoder_conf["output_size"].as<int>();
+
+        this->fsmn_layers = decoder_conf["num_blocks"].as<int>();
+        this->fsmn_lorder = decoder_conf["kernel_size"].as<int>()-1;
+
+        this->cif_threshold = predictor_conf["threshold"].as<double>();
+        this->tail_alphas = predictor_conf["tail_threshold"].as<double>();
+
+        this->asr_sample_rate = frontend_conf["fs"].as<int>();
+
+
+    }catch(exception const &e){
+        LOG(ERROR) << "Error when load argument from vad config YAML.";
+        exit(-1);
+    }
+}
+
+void SenseVoiceSmall::LoadConfigFromYaml(const char* filename){
+
+    YAML::Node config;
+    try{
+        config = YAML::LoadFile(filename);
+    }catch(exception const &e){
+        LOG(ERROR) << "Error loading file, yaml file error or not exist.";
+        exit(-1);
+    }
+
+    try{
+        YAML::Node frontend_conf = config["frontend_conf"];
+        YAML::Node encoder_conf = config["encoder_conf"];
+
+        this->window_type = frontend_conf["window"].as<string>();
+        this->n_mels = frontend_conf["n_mels"].as<int>();
+        this->frame_length = frontend_conf["frame_length"].as<int>();
+        this->frame_shift = frontend_conf["frame_shift"].as<int>();
+        this->lfr_m = frontend_conf["lfr_m"].as<int>();
+        this->lfr_n = frontend_conf["lfr_n"].as<int>();
+
+        this->encoder_size = encoder_conf["output_size"].as<int>();
+        this->fsmn_dims = encoder_conf["output_size"].as<int>();
+
+        this->asr_sample_rate = frontend_conf["fs"].as<int>();
+    }catch(exception const &e){
+        LOG(ERROR) << "Error when load argument from vad config YAML.";
+        exit(-1);
+    }
+}
+
+SenseVoiceSmall::~SenseVoiceSmall()
+{
+    if(vocab){
+        delete vocab;
+    }
+    if(online_vocab){
+        delete online_vocab;
+    }
+    if(lm_vocab){
+        delete lm_vocab;
+    }
+    if(seg_dict){
+        delete seg_dict;
+    }
+    if(phone_set_){
+        delete phone_set_;
+    }
+}
+
+void SenseVoiceSmall::StartUtterance()
+{
+}
+
+void SenseVoiceSmall::EndUtterance()
+{
+}
+
+void SenseVoiceSmall::Reset()
+{
+}
+
+void SenseVoiceSmall::FbankKaldi(float sample_rate, const float* waves, int len, std::vector<std::vector<float>> &asr_feats) {
+    knf::OnlineFbank fbank_(fbank_opts_);
+    std::vector<float> buf(len);
+    for (int32_t i = 0; i != len; ++i) {
+        buf[i] = waves[i] * 32768;
+    }
+    fbank_.AcceptWaveform(sample_rate, buf.data(), buf.size());
+
+    int32_t frames = fbank_.NumFramesReady();
+    for (int32_t i = 0; i != frames; ++i) {
+        const float *frame = fbank_.GetFrame(i);
+        std::vector<float> frame_vector(frame, frame + fbank_opts_.mel_opts.num_bins);
+        asr_feats.emplace_back(frame_vector);
+    }
+}
+
+void SenseVoiceSmall::LoadCmvn(const char *filename)
+{
+    ifstream cmvn_stream(filename);
+    if (!cmvn_stream.is_open()) {
+        LOG(ERROR) << "Failed to open file: " << filename;
+        exit(-1);
+    }
+    string line;
+
+    while (getline(cmvn_stream, line)) {
+        istringstream iss(line);
+        vector<string> line_item{istream_iterator<string>{iss}, istream_iterator<string>{}};
+        if (line_item[0] == "<AddShift>") {
+            getline(cmvn_stream, line);
+            istringstream means_lines_stream(line);
+            vector<string> means_lines{istream_iterator<string>{means_lines_stream}, istream_iterator<string>{}};
+            if (means_lines[0] == "<LearnRateCoef>") {
+                for (int j = 3; j < means_lines.size() - 1; j++) {
+                    means_list_.push_back(stof(means_lines[j]));
+                }
+                continue;
+            }
+        }
+        else if (line_item[0] == "<Rescale>") {
+            getline(cmvn_stream, line);
+            istringstream vars_lines_stream(line);
+            vector<string> vars_lines{istream_iterator<string>{vars_lines_stream}, istream_iterator<string>{}};
+            if (vars_lines[0] == "<LearnRateCoef>") {
+                for (int j = 3; j < vars_lines.size() - 1; j++) {
+                    vars_list_.push_back(stof(vars_lines[j])*scale);
+                }
+                continue;
+            }
+        }
+    }
+}
+
+string SenseVoiceSmall::CTCSearch(float * in, std::vector<int32_t> paraformer_length, std::vector<int64_t> outputShape)
+{
+    std::string unicodeChar = "▁";
+    int32_t vocab_size = outputShape[2];
+
+    std::vector<int64_t> tokens;
+    std::string text="";
+    int32_t prev_id = -1;
+    for (int32_t t = 0; t != paraformer_length[0]; ++t) {
+        auto y = std::distance(
+            static_cast<const float *>(in),
+            std::max_element(
+                static_cast<const float *>(in),
+                static_cast<const float *>(in) + vocab_size));
+        in += vocab_size;
+
+        if (y != blank_id && y != prev_id) {
+            tokens.push_back(y);
+        }
+        prev_id = y;
+    }
+    string str_lang = "";
+    string str_emo = "";
+    string str_event = "";
+    string str_itn = "";
+    if(tokens.size() >=3){
+        str_lang  = vocab->Id2String(tokens[0]);
+        str_emo   = vocab->Id2String(tokens[1]);
+        str_event = vocab->Id2String(tokens[2]);
+        str_itn = vocab->Id2String(tokens[3]);
+    }
+
+    for(int32_t i = 4; i < tokens.size(); ++i){
+        string word = vocab->Id2String(tokens[i]);
+        size_t found = word.find(unicodeChar);
+        if(found != std::string::npos){
+            text += " " + word.substr(3);
+        }else{
+            text += word;
+        }
+    }
+    if(str_itn == "<|withitn|>"){
+        if(str_lang == "<|zh|>"){
+            text += "。";
+        }else{
+            text += ".";
+        }
+    }
+
+    return str_lang + str_emo + str_event + " " + text;
+}
+
+string SenseVoiceSmall::GreedySearch(float * in, int n_len,  int64_t token_nums, bool is_stamp, std::vector<float> us_alphas, std::vector<float> us_cif_peak)
+{
+    vector<int> hyps;
+    int Tmax = n_len;
+    for (int i = 0; i < Tmax; i++) {
+        int max_idx;
+        float max_val;
+        FindMax(in + i * token_nums, token_nums, max_val, max_idx);
+        hyps.push_back(max_idx);
+    }
+    if(!is_stamp){
+        return online_vocab->Vector2StringV2(hyps, language);
+    }else{
+        std::vector<string> char_list;
+        std::vector<std::vector<float>> timestamp_list;
+        std::string res_str;
+        online_vocab->Vector2String(hyps, char_list);
+        std::vector<string> raw_char(char_list);
+        TimestampOnnx(us_alphas, us_cif_peak, char_list, res_str, timestamp_list);
+
+        return PostProcess(raw_char, timestamp_list);
+    }
+}
+
+void SenseVoiceSmall::LfrCmvn(std::vector<std::vector<float>> &asr_feats) {
+
+    std::vector<std::vector<float>> out_feats;
+    int T = asr_feats.size();
+    int T_lrf = ceil(1.0 * T / lfr_n);
+
+    // Pad frames at start(copy first frame)
+    for (int i = 0; i < (lfr_m - 1) / 2; i++) {
+        asr_feats.insert(asr_feats.begin(), asr_feats[0]);
+    }
+    // Merge lfr_m frames as one,lfr_n frames per window
+    T = T + (lfr_m - 1) / 2;
+    std::vector<float> p;
+    for (int i = 0; i < T_lrf; i++) {
+        if (lfr_m <= T - i * lfr_n) {
+            for (int j = 0; j < lfr_m; j++) {
+                p.insert(p.end(), asr_feats[i * lfr_n + j].begin(), asr_feats[i * lfr_n + j].end());
+            }
+            out_feats.emplace_back(p);
+            p.clear();
+        } else {
+            // Fill to lfr_m frames at last window if less than lfr_m frames  (copy last frame)
+            int num_padding = lfr_m - (T - i * lfr_n);
+            for (int j = 0; j < (asr_feats.size() - i * lfr_n); j++) {
+                p.insert(p.end(), asr_feats[i * lfr_n + j].begin(), asr_feats[i * lfr_n + j].end());
+            }
+            for (int j = 0; j < num_padding; j++) {
+                p.insert(p.end(), asr_feats[asr_feats.size() - 1].begin(), asr_feats[asr_feats.size() - 1].end());
+            }
+            out_feats.emplace_back(p);
+            p.clear();
+        }
+    }
+    // Apply cmvn
+    for (auto &out_feat: out_feats) {
+        for (int j = 0; j < means_list_.size(); j++) {
+            out_feat[j] = (out_feat[j] + means_list_[j]) * vars_list_[j];
+        }
+    }
+    asr_feats = out_feats;
+}
+
+std::vector<std::vector<float>> SenseVoiceSmall::CompileHotwordEmbedding(std::string &hotwords) {
+    int embedding_dim = encoder_size;
+    std::vector<std::vector<float>> hw_emb;
+    std::vector<float> vec(embedding_dim, 0);
+    hw_emb.push_back(vec);
+    return hw_emb;
+}
+
+std::vector<std::string> SenseVoiceSmall::Forward(float** din, int* len, bool input_finished, std::string svs_lang, bool svs_itn, int batch_in)
+{
+    std::vector<std::string> results;
+    string result="";
+    int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins;
+
+    if(batch_in != 1){
+        results.push_back(result);
+        return results;
+    }
+
+    std::vector<std::vector<float>> asr_feats;
+    FbankKaldi(asr_sample_rate, din[0], len[0], asr_feats);
+    if(asr_feats.size() == 0){
+        results.push_back(result);
+        return results;
+    }
+    LfrCmvn(asr_feats);
+    int32_t feat_dim = lfr_m*in_feat_dim;
+    int32_t num_frames = asr_feats.size();
+
+    std::vector<float> wav_feats;
+    for (const auto &frame_feat: asr_feats) {
+        wav_feats.insert(wav_feats.end(), frame_feat.begin(), frame_feat.end());
+    }
+
+    //lid textnorm
+    int svs_lid = 0;
+    int svs_itnid = 15;
+    if(lid_map.find(svs_lang) != lid_map.end()){
+        svs_lid = lid_map[svs_lang];
+    }
+    if(svs_itn){
+        svs_itnid = 14;
+    }
+
+#ifdef _WIN_X86
+        Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
+#else
+        Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
+#endif
+
+    const int64_t input_shape_[3] = {1, num_frames, feat_dim};
+    Ort::Value onnx_feats = Ort::Value::CreateTensor<float>(m_memoryInfo,
+        wav_feats.data(),
+        wav_feats.size(),
+        input_shape_,
+        3);
+
+    const int64_t paraformer_length_shape[1] = {1};
+    std::vector<int32_t> paraformer_length;
+    paraformer_length.emplace_back(num_frames);
+    Ort::Value onnx_feats_len = Ort::Value::CreateTensor<int32_t>(
+          m_memoryInfo, paraformer_length.data(), paraformer_length.size(), paraformer_length_shape, 1);
+
+    const int64_t lid_shape[1] = {1};
+    std::vector<int32_t> lid_length;
+    lid_length.emplace_back(svs_lid);
+    Ort::Value onnx_lid = Ort::Value::CreateTensor<int32_t>(
+          m_memoryInfo, lid_length.data(), lid_length.size(), lid_shape, 1);
+
+    const int64_t textnorm_shape[1] = {1};
+    std::vector<int32_t> textnorm_length;
+    textnorm_length.emplace_back(svs_itnid);
+    Ort::Value onnx_itn = Ort::Value::CreateTensor<int32_t>(
+          m_memoryInfo, textnorm_length.data(), textnorm_length.size(), textnorm_shape, 1);
+
+    std::vector<Ort::Value> input_onnx;
+    input_onnx.emplace_back(std::move(onnx_feats));
+    input_onnx.emplace_back(std::move(onnx_feats_len));
+    input_onnx.emplace_back(std::move(onnx_lid));
+    input_onnx.emplace_back(std::move(onnx_itn));
+
+    try {
+        auto outputTensor = m_session_->Run(Ort::RunOptions{nullptr}, m_szInputNames.data(), input_onnx.data(), input_onnx.size(), m_szOutputNames.data(), m_szOutputNames.size());
+        float* floatData = outputTensor[0].GetTensorMutableData<float>();
+        std::vector<int64_t> outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape();
+
+        result = CTCSearch(floatData, paraformer_length, outputShape);
+    }
+    catch (std::exception const &e)
+    {
+        LOG(ERROR)<<e.what();
+    }
+
+    results.push_back(result);
+    return results;
+}
+
+string SenseVoiceSmall::Rescoring()
+{
+    LOG(ERROR)<<"Not Imp!!!!!!";
+    return "";
+}
+} // namespace funasr

runtime/onnxruntime/src/sensevoice-small.h

@@ -0,0 +1,122 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+#pragma once
+
+#include "precomp.h"
+#include "phone-set.h"
+
+namespace funasr {
+
+    class SenseVoiceSmall : public Model {
+    private:
+        Vocab* vocab = nullptr;
+        Vocab* online_vocab = nullptr;
+        Vocab* lm_vocab = nullptr;
+        SegDict* seg_dict = nullptr;
+        PhoneSet* phone_set_ = nullptr;
+        const float scale = 1.0;
+
+        void LoadConfigFromYaml(const char* filename);
+        void LoadOnlineConfigFromYaml(const char* filename);
+        void LoadCmvn(const char *filename);
+        void LfrCmvn(std::vector<std::vector<float>> &asr_feats);
+
+        std::shared_ptr<Ort::Session> hw_m_session = nullptr;
+        Ort::Env hw_env_;
+        Ort::SessionOptions hw_session_options;
+        vector<string> hw_m_strInputNames, hw_m_strOutputNames;
+        vector<const char*> hw_m_szInputNames;
+        vector<const char*> hw_m_szOutputNames;
+        bool use_hotword;
+
+    public:
+        SenseVoiceSmall();
+        ~SenseVoiceSmall();
+        void InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);
+        // online
+        void InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);
+        // 2pass
+        void InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, 
+            const std::string &token_file, const std::string &online_token_file, int thread_num);
+        // void InitHwCompiler(const std::string &hw_model, int thread_num);
+        // void InitSegDict(const std::string &seg_dict_model);
+        std::vector<std::vector<float>> CompileHotwordEmbedding(std::string &hotwords);
+        void Reset();
+        void FbankKaldi(float sample_rate, const float* waves, int len, std::vector<std::vector<float>> &asr_feats);
+        std::vector<std::string> Forward(float** din, int* len, bool input_finished=true, std::string svs_lang="auto", bool svs_itn=true, int batch_in=1);
+        string CTCSearch( float * in, std::vector<int32_t> paraformer_length, std::vector<int64_t> outputShape);
+        string GreedySearch( float* in, int n_len, int64_t token_nums,
+                             bool is_stamp=false, std::vector<float> us_alphas={0}, std::vector<float> us_cif_peak={0});
+        string Rescoring();
+        string GetLang(){return language;};
+        int GetAsrSampleRate() { return asr_sample_rate; };
+        int GetBatchSize() {return batch_size_;};
+        void StartUtterance();
+        void EndUtterance();
+        // void InitLm(const std::string &lm_file, const std::string &lm_cfg_file, const std::string &lex_file);
+        // string BeamSearch(WfstDecoder* &wfst_decoder, float* in, int n_len, int64_t token_nums);
+        // string FinalizeDecode(WfstDecoder* &wfst_decoder,
+        //                   bool is_stamp=false, std::vector<float> us_alphas={0}, std::vector<float> us_cif_peak={0});
+        // Vocab* GetVocab();
+        // Vocab* GetLmVocab();
+        // PhoneSet* GetPhoneSet();
+		
+        knf::FbankOptions fbank_opts_;
+        vector<float> means_list_;
+        vector<float> vars_list_;
+        int lfr_m = PARA_LFR_M;
+        int lfr_n = PARA_LFR_N;
+
+        // paraformer-offline
+        std::shared_ptr<Ort::Session> m_session_ = nullptr;
+        Ort::Env env_;
+        Ort::SessionOptions session_options_;
+
+        vector<string> m_strInputNames, m_strOutputNames;
+        vector<const char*> m_szInputNames;
+        vector<const char*> m_szOutputNames;
+
+        std::string language="zh-cn";
+
+        // paraformer-online
+        std::shared_ptr<Ort::Session> encoder_session_ = nullptr;
+        std::shared_ptr<Ort::Session> decoder_session_ = nullptr;
+        vector<string> en_strInputNames, en_strOutputNames;
+        vector<const char*> en_szInputNames_;
+        vector<const char*> en_szOutputNames_;
+        vector<string> de_strInputNames, de_strOutputNames;
+        vector<const char*> de_szInputNames_;
+        vector<const char*> de_szOutputNames_;
+
+        // lm
+        std::shared_ptr<fst::Fst<fst::StdArc>> lm_ = nullptr;
+
+        string window_type = "hamming";
+        int frame_length = 25;
+        int frame_shift = 10;
+        int n_mels = 80;
+        int encoder_size = 512;
+        int fsmn_layers = 16;
+        int fsmn_lorder = 10;
+        int fsmn_dims = 512;
+        int asr_sample_rate = MODEL_SAMPLE_RATE;
+        int batch_size_ = 1;
+        int blank_id = 0;
+        float cif_threshold = 1.0;
+        float tail_alphas = 0.45;
+        //dict
+        std::map<std::string, int> lid_map = {
+            {"auto", 0},
+            {"zh", 3},
+            {"en", 4},
+            {"yue", 7},
+            {"ja", 11},
+            {"ko", 12},
+            {"nospeech", 13}
+        };
+        
+    };
+
+} // namespace funasr

runtime/onnxruntime/src/tensor.h

@@ -0,0 +1,161 @@
+#ifndef TENSOR_H
+#define TENSOR_H
+
+#include "alignedmem.h"
+#include "stdio.h"
+#include <iostream>
+#include <cstring>
+using namespace std;
+
+namespace funasr {
+
+template <typename T> class Tensor {
+  private:
+    void alloc_buff();
+    void free_buff();
+    int mem_size;
+
+  public:
+    T *buff;
+    int size[4];
+    int buff_size;
+    Tensor(Tensor<T> *in);
+    Tensor(int a);
+    Tensor(int a, int b);
+    Tensor(int a, int b, int c);
+    Tensor(int a, int b, int c, int d);
+    ~Tensor();
+    void zeros();
+    void shape();
+    void disp();
+    void dump(const char *mode);
+    void concat(Tensor<T> *din, int dim);
+    void resize(int a, int b, int c, int d);
+    void add(float coe, Tensor<T> *in);
+    void add(Tensor<T> *in);
+    void add(Tensor<T> *in1, Tensor<T> *in2);
+    void reload(Tensor<T> *in);
+};
+
+template <typename T> Tensor<T>::Tensor(int a) : size{1, 1, 1, a}
+{
+    alloc_buff();
+}
+
+template <typename T> Tensor<T>::Tensor(int a, int b) : size{1, 1, a, b}
+{
+    alloc_buff();
+}
+
+template <typename T> Tensor<T>::Tensor(int a, int b, int c) : size{1, a, b, c}
+{
+
+    alloc_buff();
+}
+
+template <typename T>
+Tensor<T>::Tensor(int a, int b, int c, int d) : size{a, b, c, d}
+{
+    alloc_buff();
+}
+
+template <typename T> Tensor<T>::Tensor(Tensor<T> *in)
+{
+    memcpy(size, in->size, 4 * sizeof(int));
+    alloc_buff();
+    memcpy(buff, in->buff, in->buff_size * sizeof(T));
+}
+
+template <typename T> Tensor<T>::~Tensor()
+{
+    free_buff();
+}
+
+template <typename T> void Tensor<T>::alloc_buff()
+{
+    buff_size = size[0] * size[1] * size[2] * size[3];
+    mem_size = buff_size;
+    buff = (T *)AlignedMalloc(32, buff_size * sizeof(T));
+}
+
+template <typename T> void Tensor<T>::free_buff()
+{
+    aligned_free(buff);
+}
+
+template <typename T> void Tensor<T>::zeros()
+{
+    memset(buff, 0, buff_size * sizeof(T));
+}
+
+template <typename T> void Tensor<T>::shape()
+{
+    printf("(%d,%d,%d,%d)\n", size[0], size[1], size[2], size[3]);
+}
+
+// TODO:: fix it!!!!
+template <typename T> void Tensor<T>::concat(Tensor<T> *din, int dim)
+{
+    memcpy(buff + buff_size, din->buff, din->buff_size * sizeof(T));
+    buff_size += din->buff_size;
+    size[dim] += din->size[dim];
+}
+
+// TODO:: fix it!!!!
+template <typename T> void Tensor<T>::resize(int a, int b, int c, int d)
+{
+    size[0] = a;
+    size[1] = b;
+    size[2] = c;
+    size[3] = d;
+    buff_size = size[0] * size[1] * size[2] * size[3];
+}
+
+template <typename T> void Tensor<T>::add(float coe, Tensor<T> *in)
+{
+    int i;
+    for (i = 0; i < buff_size; i++) {
+        buff[i] = buff[i] + coe * in->buff[i];
+    }
+}
+
+template <typename T> void Tensor<T>::add(Tensor<T> *in)
+{
+    int i;
+    for (i = 0; i < buff_size; i++) {
+        buff[i] = buff[i] + in->buff[i];
+    }
+}
+
+template <typename T> void Tensor<T>::add(Tensor<T> *in1, Tensor<T> *in2)
+{
+    int i;
+    for (i = 0; i < buff_size; i++) {
+        buff[i] = buff[i] + in1->buff[i] + in2->buff[i];
+    }
+}
+
+template <typename T> void Tensor<T>::reload(Tensor<T> *in)
+{
+    memcpy(buff, in->buff, in->buff_size * sizeof(T));
+}
+
+template <typename T> void Tensor<T>::disp()
+{
+    int i;
+    for (i = 0; i < buff_size; i++) {
+        cout << buff[i] << " ";
+    }
+    cout << endl;
+}
+
+template <typename T> void Tensor<T>::dump(const char *mode)
+{
+    FILE *fp;
+    fp = fopen("tmp.bin", mode);
+    fwrite(buff, 1, buff_size * sizeof(T), fp);
+    fclose(fp);
+}
+
+} // namespace funasr
+#endif

runtime/onnxruntime/src/tokenizer.cpp

@@ -0,0 +1,351 @@
+ /**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#include "precomp.h"
+
+namespace funasr {
+CTokenizer::CTokenizer(const char* sz_yamlfile):m_ready(false)
+{
+	OpenYaml(sz_yamlfile);
+}
+
+CTokenizer::CTokenizer():m_ready(false)
+{
+}
+
+CTokenizer::~CTokenizer()
+{
+	if (jieba_dict_trie_){
+		delete jieba_dict_trie_;
+	}
+	if (jieba_model_){
+    	delete jieba_model_;
+	}
+}
+
+void CTokenizer::SetJiebaRes(cppjieba::DictTrie *dict, cppjieba::HMMModel *hmm) {
+	jieba_processor_.SetJiebaRes(dict, hmm);
+}
+
+void CTokenizer::JiebaInit(std::string punc_config){
+    if (seg_jieba){
+        std::string model_path = punc_config.substr(0, punc_config.length() - (sizeof(PUNC_CONFIG_NAME)-1));
+        std::string jieba_dict_file = PathAppend(model_path, JIEBA_DICT);
+        std::string jieba_hmm_file = PathAppend(model_path, JIEBA_HMM_MODEL);
+        std::string jieba_userdict_file = PathAppend(model_path, JIEBA_USERDICT);
+		try{
+        	jieba_dict_trie_ = new cppjieba::DictTrie(jieba_dict_file, jieba_userdict_file);
+			LOG(INFO) << "Successfully load file from " << jieba_dict_file << ", " << jieba_userdict_file;
+		}catch(exception const &e){
+			LOG(ERROR) << "Error loading file, Jieba dict file error or not exist.";
+			exit(-1);
+		}
+
+		try{
+        	jieba_model_ = new cppjieba::HMMModel(jieba_hmm_file);
+			LOG(INFO) << "Successfully load model from " << jieba_hmm_file;
+		}catch(exception const &e){
+			LOG(ERROR) << "Error loading file, Jieba hmm file error or not exist.";
+			exit(-1);
+		}
+
+        SetJiebaRes(jieba_dict_trie_, jieba_model_);
+    }else {
+        jieba_dict_trie_ = nullptr;
+        jieba_model_ = nullptr;
+    }
+}
+
+void CTokenizer::ReadYaml(const YAML::Node& node) 
+{
+	if (node.IsMap()) 
+	{//��map��
+		for (auto it = node.begin(); it != node.end(); ++it) 
+		{
+			ReadYaml(it->second);
+		}
+	}
+	if (node.IsSequence()) {//��������
+		for (size_t i = 0; i < node.size(); ++i) {
+			ReadYaml(node[i]);
+		}
+	}
+	if (node.IsScalar()) {//�DZ�����
+		LOG(INFO) << node.as<string>();
+	}
+}
+
+bool CTokenizer::OpenYaml(const char* sz_yamlfile)
+{
+	YAML::Node m_Config;
+	try{
+		m_Config = YAML::LoadFile(sz_yamlfile);
+	}catch(exception const &e){
+        LOG(INFO) << "Error loading file, yaml file error or not exist.";
+        exit(-1);
+    }
+
+	try
+	{
+		YAML::Node conf_seg_jieba = m_Config["seg_jieba"];
+        if (conf_seg_jieba.IsDefined()){
+            seg_jieba = conf_seg_jieba.as<bool>();
+        }
+
+		auto Tokens = m_Config["token_list"];
+		if (Tokens.IsSequence())
+		{
+			for (size_t i = 0; i < Tokens.size(); ++i) 
+			{
+				if (Tokens[i].IsScalar())
+				{
+					m_id2token.push_back(Tokens[i].as<string>());
+					m_token2id.insert(make_pair<string, int>(Tokens[i].as<string>(), i));
+				}
+			}
+		}
+		auto Puncs = m_Config["punc_list"];
+		if (Puncs.IsSequence())
+		{
+			for (size_t i = 0; i < Puncs.size(); ++i)
+			{
+				if (Puncs[i].IsScalar())
+				{ 
+					m_id2punc.push_back(Puncs[i].as<string>());
+					m_punc2id.insert(make_pair<string, int>(Puncs[i].as<string>(), i));
+				}
+			}
+		}
+	}
+	catch (YAML::BadFile& e) {
+		LOG(ERROR) << "Read error!";
+		return  false;
+	}
+	m_ready = true;
+	return m_ready;
+}
+
+bool CTokenizer::OpenYaml(const char* sz_yamlfile, const char* token_file)
+{
+	YAML::Node m_Config;
+	try{
+		m_Config = YAML::LoadFile(sz_yamlfile);
+	}catch(exception const &e){
+        LOG(INFO) << "Error loading file, yaml file error or not exist.";
+        exit(-1);
+    }
+
+	try
+	{
+		YAML::Node conf_seg_jieba = m_Config["seg_jieba"];
+        if (conf_seg_jieba.IsDefined()){
+            seg_jieba = conf_seg_jieba.as<bool>();
+        }
+
+		auto Puncs = m_Config["model_conf"]["punc_list"];
+		if (Puncs.IsSequence())
+		{
+			for (size_t i = 0; i < Puncs.size(); ++i)
+			{
+				if (Puncs[i].IsScalar())
+				{ 
+					m_id2punc.push_back(Puncs[i].as<string>());
+					m_punc2id.insert(make_pair<string, int>(Puncs[i].as<string>(), i));
+				}
+			}
+		}
+
+		nlohmann::json json_array;
+		std::ifstream file(token_file);
+		if (file.is_open()) {
+			file >> json_array;
+			file.close();
+		} else {
+			LOG(INFO) << "Error loading token file, token file error or not exist.";
+			return  false;
+		}
+
+		int i = 0;
+		for (const auto& element : json_array) {
+			m_id2token.push_back(element);
+			m_token2id[element] = i;
+			i++;
+		}
+	}
+	catch (YAML::BadFile& e) {
+		LOG(ERROR) << "Read error!";
+		return  false;
+	}
+	m_ready = true;
+	return m_ready;
+}
+
+vector<string> CTokenizer::Id2String(vector<int> input)
+{
+	vector<string> result;
+	for (auto& item : input)
+	{
+		result.push_back(m_id2token[item]);
+	}
+	return result;
+}
+
+int CTokenizer::String2Id(string input)
+{
+	int nID = 0; // <blank>
+	if (m_token2id.find(input) != m_token2id.end())
+		nID=(m_token2id[input]);
+	else
+		nID=(m_token2id[UNK_CHAR]);
+	return nID;
+}
+
+vector<int> CTokenizer::String2Ids(vector<string> input)
+{
+	vector<int> result;
+	for (auto& item : input)
+	{	
+		transform(item.begin(), item.end(), item.begin(), ::tolower);
+		if (m_token2id.find(item) != m_token2id.end())
+			result.push_back(m_token2id[item]);
+		else
+			result.push_back(m_token2id[UNK_CHAR]);
+	}
+	return result;
+}
+
+vector<string> CTokenizer::Id2Punc(vector<int> input)
+{
+	vector<string> result;
+	for (auto& item : input)
+	{
+		result.push_back(m_id2punc[item]);
+	}
+	return result;
+}
+
+string CTokenizer::Id2Punc(int n_punc_id)
+{
+	return m_id2punc[n_punc_id];
+}
+
+vector<int> CTokenizer::Punc2Ids(vector<string> input)
+{
+	vector<int> result;
+	for (auto& item : input)
+	{
+		result.push_back(m_punc2id[item]);
+	}
+	return result;
+}
+
+bool CTokenizer::IsPunc(string& Punc)
+{
+	if (m_punc2id.find(Punc) != m_punc2id.end())
+		return true;
+	else
+		return false;
+}
+
+vector<string> CTokenizer::SplitChineseString(const string & str_info)
+{
+	vector<string> list;
+	int strSize = str_info.size();
+	int i = 0;
+
+	while (i < strSize) {
+		int len = 1;
+		for (int j = 0; j < 6 && (str_info[i] & (0x80 >> j)); j++) {
+			len = j + 1;
+		}
+		list.push_back(str_info.substr(i, len));
+		i += len;
+	}
+	return list;
+}
+
+vector<string> CTokenizer::SplitChineseJieba(const string & str_info)
+{
+	vector<string> list;
+	jieba_processor_.Cut(str_info, list, false);
+
+	return list;
+}
+
+void CTokenizer::StrSplit(const string& str, const char split, vector<string>& res)
+{
+	if (str == "")
+	{
+		return;
+	}
+	string&& strs = str + split;
+	size_t pos = strs.find(split);
+
+	while (pos != string::npos)
+	{
+		res.emplace_back(strs.substr(0, pos));
+		strs = move(strs.substr(pos + 1, strs.size()));
+		pos = strs.find(split);
+	}
+}
+
+void CTokenizer::Tokenize(const char* str_info, vector<string> & str_out, vector<int> & id_out)
+{
+	vector<string>  strList;
+	StrSplit(str_info,' ', strList);
+	string current_eng,current_chinese;
+	for (auto& item : strList)
+	{
+		current_eng = "";
+		current_chinese = "";
+		for (auto& ch : item)
+		{
+			if (!(ch& 0x80))
+			{ // Ӣ��
+				if (current_chinese.size() > 0)
+				{
+					// for utf-8 chinese
+					vector<string> chineseList;
+					if(seg_jieba){
+						chineseList = SplitChineseJieba(current_chinese);
+					}else{
+						chineseList = SplitChineseString(current_chinese);
+					}
+					str_out.insert(str_out.end(), chineseList.begin(),chineseList.end());
+					current_chinese = "";
+				}
+				current_eng += ch;
+			}
+			else
+			{
+				if (current_eng.size() > 0)
+				{
+					str_out.push_back(current_eng);
+					current_eng = "";
+				}
+				current_chinese += ch;
+			}
+		}
+		if (current_chinese.size() > 0)
+		{
+			// for utf-8 chinese
+			vector<string> chineseList;
+			if(seg_jieba){
+				chineseList = SplitChineseJieba(current_chinese);
+			}else{
+				chineseList = SplitChineseString(current_chinese);
+			}
+			str_out.insert(str_out.end(), chineseList.begin(), chineseList.end());
+			current_chinese = "";
+		}
+		if (current_eng.size() > 0)
+		{
+			str_out.push_back(current_eng);
+		}
+	}
+	id_out= String2Ids(str_out);
+}
+
+} // namespace funasr

runtime/onnxruntime/src/tokenizer.h

@@ -0,0 +1,49 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License  (https://opensource.org/licenses/MIT)
+*/
+
+#pragma once
+#include <yaml-cpp/yaml.h>
+#include "cppjieba/DictTrie.hpp"
+#include "cppjieba/HMMModel.hpp"
+#include "cppjieba/Jieba.hpp"
+#include "nlohmann/json.hpp"
+
+namespace funasr {
+class CTokenizer {
+private:
+
+	bool  m_ready = false;
+	vector<string>   m_id2token,m_id2punc;
+	map<string, int>  m_token2id,m_punc2id;
+
+	cppjieba::DictTrie *jieba_dict_trie_=nullptr;
+    cppjieba::HMMModel *jieba_model_=nullptr;
+	cppjieba::Jieba jieba_processor_;
+
+public:
+
+	CTokenizer(const char* sz_yamlfile);
+	CTokenizer();
+	~CTokenizer();
+	bool OpenYaml(const char* sz_yamlfile);
+	bool OpenYaml(const char* sz_yamlfile, const char* token_file);
+	void ReadYaml(const YAML::Node& node);
+	vector<string> Id2String(vector<int> input);
+	vector<int> String2Ids(vector<string> input);
+	int String2Id(string input);
+	vector<string> Id2Punc(vector<int> input);
+	string Id2Punc(int n_punc_id);
+	vector<int> Punc2Ids(vector<string> input);
+	vector<string> SplitChineseString(const string& str_info);
+	vector<string> SplitChineseJieba(const string& str_info);
+	void StrSplit(const string& str, const char split, vector<string>& res);
+	void Tokenize(const char* str_info, vector<string>& str_out, vector<int>& id_out);
+	bool IsPunc(string& Punc);
+	bool seg_jieba = false;
+	void SetJiebaRes(cppjieba::DictTrie *dict, cppjieba::HMMModel *hmm);
+	void JiebaInit(std::string punc_config);
+};
+
+} // namespace funasr

runtime/onnxruntime/src/tpass-online-stream.cpp

@@ -0,0 +1,28 @@
+#include "precomp.h"
+
+namespace funasr {
+TpassOnlineStream::TpassOnlineStream(TpassStream* tpass_stream, std::vector<int> chunk_size){
+    TpassStream* tpass_obj = (TpassStream*)tpass_stream;
+    if(tpass_obj->vad_handle){
+        vad_online_handle = make_unique<FsmnVadOnline>((FsmnVad*)(tpass_obj->vad_handle).get());
+    }else{
+        LOG(ERROR)<<"vad_handle is null";
+        exit(-1);
+    }
+
+    if(tpass_obj->asr_handle){
+        asr_online_handle = make_unique<ParaformerOnline>((tpass_obj->asr_handle).get(), chunk_size, tpass_stream->GetModelType());
+    }else{
+        LOG(ERROR)<<"asr_handle is null";
+        exit(-1);
+    }
+}
+
+TpassOnlineStream* CreateTpassOnlineStream(void* tpass_stream, std::vector<int> chunk_size)
+{
+    TpassOnlineStream *mm;
+    mm =new TpassOnlineStream((TpassStream*)tpass_stream, chunk_size);
+    return mm;
+}
+
+} // namespace funasr

runtime/onnxruntime/src/tpass-stream.cpp

@@ -0,0 +1,144 @@
+#include "precomp.h"
+
+namespace funasr {
+TpassStream::TpassStream(std::map<std::string, std::string>& model_path, int thread_num)
+{
+    // VAD model
+    if(model_path.find(VAD_DIR) != model_path.end()){
+        string vad_model_path;
+        string vad_cmvn_path;
+        string vad_config_path;
+    
+        vad_model_path = PathAppend(model_path.at(VAD_DIR), MODEL_NAME);
+        if(model_path.find(VAD_QUANT) != model_path.end() && model_path.at(VAD_QUANT) == "true"){
+            vad_model_path = PathAppend(model_path.at(VAD_DIR), QUANT_MODEL_NAME);
+        }
+        vad_cmvn_path = PathAppend(model_path.at(VAD_DIR), VAD_CMVN_NAME);
+        vad_config_path = PathAppend(model_path.at(VAD_DIR), VAD_CONFIG_NAME);
+        if (access(vad_model_path.c_str(), F_OK) != 0 ||
+            access(vad_cmvn_path.c_str(), F_OK) != 0 ||
+            access(vad_config_path.c_str(), F_OK) != 0 )
+        {
+            LOG(INFO) << "VAD model file is not exist, skip load vad model.";
+        }else{
+            vad_handle = make_unique<FsmnVad>();
+            vad_handle->InitVad(vad_model_path, vad_cmvn_path, vad_config_path, thread_num);
+            use_vad = true;
+        }
+    }
+
+    // AM model
+    if(model_path.find(OFFLINE_MODEL_DIR) != model_path.end() && model_path.find(ONLINE_MODEL_DIR) != model_path.end()){
+        // 2pass
+        string am_model_path;
+        string en_model_path;
+        string de_model_path;
+        string am_cmvn_path;
+        string am_config_path;
+        string token_path;
+        string online_token_path;
+        string hw_compile_model_path;
+        string seg_dict_path;
+        
+        if (model_path.at(MODEL_DIR).find(MODEL_SVS) != std::string::npos)
+        {
+            asr_handle = make_unique<SenseVoiceSmall>();
+            model_type = MODEL_SVS;
+        }else{
+            asr_handle = make_unique<Paraformer>();
+        }
+
+        bool enable_hotword = false;
+        hw_compile_model_path = PathAppend(model_path.at(MODEL_DIR), MODEL_EB_NAME);
+        seg_dict_path = PathAppend(model_path.at(MODEL_DIR), MODEL_SEG_DICT);
+        if ((access(hw_compile_model_path.c_str(), F_OK) == 0) && 
+            (access(seg_dict_path.c_str(), F_OK) == 0)) { // if model_eb.onnx exist, hotword enabled
+          enable_hotword = true;
+          asr_handle->InitHwCompiler(hw_compile_model_path, thread_num);
+          asr_handle->InitSegDict(seg_dict_path);
+        }
+
+        am_model_path = PathAppend(model_path.at(OFFLINE_MODEL_DIR), MODEL_NAME);
+        en_model_path = PathAppend(model_path.at(ONLINE_MODEL_DIR), ENCODER_NAME);
+        de_model_path = PathAppend(model_path.at(ONLINE_MODEL_DIR), DECODER_NAME);
+        online_token_path = PathAppend(model_path.at(ONLINE_MODEL_DIR), TOKEN_PATH);
+        if(model_path.find(QUANTIZE) != model_path.end() && model_path.at(QUANTIZE) == "true"){
+            am_model_path = PathAppend(model_path.at(OFFLINE_MODEL_DIR), QUANT_MODEL_NAME);
+            en_model_path = PathAppend(model_path.at(ONLINE_MODEL_DIR), QUANT_ENCODER_NAME);
+            de_model_path = PathAppend(model_path.at(ONLINE_MODEL_DIR), QUANT_DECODER_NAME);
+        }
+        am_cmvn_path = PathAppend(model_path.at(ONLINE_MODEL_DIR), AM_CMVN_NAME);
+        am_config_path = PathAppend(model_path.at(ONLINE_MODEL_DIR), AM_CONFIG_NAME);
+        token_path = PathAppend(model_path.at(MODEL_DIR), TOKEN_PATH);
+
+        asr_handle->InitAsr(am_model_path, en_model_path, de_model_path, am_cmvn_path, am_config_path, token_path, online_token_path, thread_num);
+    }else{
+        LOG(ERROR) <<"Can not find offline-model-dir or online-model-dir";
+        exit(-1);
+    }
+    
+    // Lm resource
+    if (model_path.find(LM_DIR) != model_path.end() && model_path.at(LM_DIR) != "") {
+        string fst_path, lm_config_path, lex_path;
+        fst_path = PathAppend(model_path.at(LM_DIR), LM_FST_RES);
+        lm_config_path = PathAppend(model_path.at(LM_DIR), LM_CONFIG_NAME);
+        lex_path = PathAppend(model_path.at(LM_DIR), LEX_PATH);
+        if (access(lex_path.c_str(), F_OK) != 0 )
+        {
+            LOG(ERROR) << "Lexicon.txt file is not exist, please use the latest version. Skip load LM model.";
+        }else{
+            asr_handle->InitLm(fst_path, lm_config_path, lex_path);
+        }
+    }
+
+    // PUNC model
+    if(model_path.find(PUNC_DIR) != model_path.end()){
+        string punc_model_path;
+        string punc_config_path;
+        string token_path;
+    
+        punc_model_path = PathAppend(model_path.at(PUNC_DIR), MODEL_NAME);
+        if(model_path.find(PUNC_QUANT) != model_path.end() && model_path.at(PUNC_QUANT) == "true"){
+            punc_model_path = PathAppend(model_path.at(PUNC_DIR), QUANT_MODEL_NAME);
+        }
+        punc_config_path = PathAppend(model_path.at(PUNC_DIR), PUNC_CONFIG_NAME);
+        token_path = PathAppend(model_path.at(PUNC_DIR), TOKEN_PATH);
+
+        if (access(punc_model_path.c_str(), F_OK) != 0 ||
+            access(punc_config_path.c_str(), F_OK) != 0 ||
+            access(token_path.c_str(), F_OK) != 0)
+        {
+            LOG(INFO) << "PUNC model file is not exist, skip load punc model.";
+        }else{
+            punc_online_handle = make_unique<CTTransformerOnline>();
+            punc_online_handle->InitPunc(punc_model_path, punc_config_path, token_path, thread_num);
+            use_punc = true;
+        }
+    }
+#if !defined(__APPLE__)
+    // Optional: ITN, here we just support language_type=MandarinEnglish
+    if(model_path.find(ITN_DIR) != model_path.end()){
+        string itn_tagger_path = PathAppend(model_path.at(ITN_DIR), ITN_TAGGER_NAME);
+        string itn_verbalizer_path = PathAppend(model_path.at(ITN_DIR), ITN_VERBALIZER_NAME);
+
+        if (access(itn_tagger_path.c_str(), F_OK) != 0 ||
+            access(itn_verbalizer_path.c_str(), F_OK) != 0 )
+        {
+            LOG(INFO) << "ITN model file is not exist, skip load ITN model.";
+        }else{
+            itn_handle = make_unique<ITNProcessor>();
+            itn_handle->InitITN(itn_tagger_path, itn_verbalizer_path, thread_num);
+            use_itn = true;
+        }
+    }
+#endif
+      
+}
+
+TpassStream *CreateTpassStream(std::map<std::string, std::string>& model_path, int thread_num)
+{
+    TpassStream *mm;
+    mm = new TpassStream(model_path, thread_num);
+    return mm;
+}
+} // namespace funasr

runtime/onnxruntime/src/utf8-string.cpp

@@ -0,0 +1,92 @@
+// Acknowledgement: this code is adapted from 
+// https://github.com/wenet-e2e/WeTextProcessing/blob/master/runtime/utils/string.cc
+// Retrieved in Aug 2023.
+
+// Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+//               2023 Jing Du (thuduj12@163.com)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "utf8-string.h"
+#include <glog/logging.h>
+
+namespace funasr {
+const char* WHITESPACE = " \n\r\t\f\v";
+
+int char_length(char ch) {
+  int num_bytes = 1;
+  CHECK_LE((ch & 0xF8), 0xF0);
+  if ((ch & 0x80) == 0x00) {
+    // The first 128 characters (US-ASCII) in UTF-8 format only need one byte.
+    num_bytes = 1;
+  } else if ((ch & 0xE0) == 0xC0) {
+    // The next 1,920 characters need two bytes to encode,
+    // which covers the remainder of almost all Latin-script alphabets.
+    num_bytes = 2;
+  } else if ((ch & 0xF0) == 0xE0) {
+    // Three bytes are needed for characters in the rest of
+    // the Basic Multilingual Plane, which contains virtually all characters
+    // in common use, including most Chinese, Japanese and Korean characters.
+    num_bytes = 3;
+  } else if ((ch & 0xF8) == 0xF0) {
+    // Four bytes are needed for characters in the other planes of Unicode,
+    // which include less common CJK characters, various historic scripts,
+    // mathematical symbols, and emoji (pictographic symbols).
+    num_bytes = 4;
+  }
+  return num_bytes;
+}
+
+int string_length(const std::string& str) {
+  int len = 0;
+  int num_bytes = 1;
+  for (size_t i = 0; i < str.length(); i += num_bytes) {
+    num_bytes = char_length(str[i]);
+    ++len;
+  }
+  return len;
+}
+
+void string2chars(const std::string& str, std::vector<std::string>* chars) {
+  chars->clear();
+  int num_bytes = 1;
+  for (size_t i = 0; i < str.length(); i += num_bytes) {
+    num_bytes = char_length(str[i]);
+    chars->push_back(str.substr(i, num_bytes));
+  }
+}
+
+std::string ltrim(const std::string& str) {
+  size_t start = str.find_first_not_of(WHITESPACE);
+  return (start == std::string::npos) ? "" : str.substr(start);
+}
+
+std::string rtrim(const std::string& str) {
+  size_t end = str.find_last_not_of(WHITESPACE);
+  return end == std::string::npos ? "" : str.substr(0, end + 1);
+}
+
+std::string trim(const std::string& str) { return rtrim(ltrim(str)); }
+
+void split_string(const std::string& str, const std::string& delim,
+                  std::vector<std::string>* output) {
+  std::string s = str;
+  size_t pos = 0;
+  while ((pos = s.find(delim)) != std::string::npos) {
+    output->emplace_back(s.substr(0, pos));
+    s.erase(0, pos + delim.length());
+  }
+  output->emplace_back(s);
+}
+
+}  // namespace funasr

runtime/onnxruntime/src/utf8-string.h

@@ -0,0 +1,46 @@
+// Acknowledgement: this code is adapted from 
+// https://github.com/wenet-e2e/WeTextProcessing/blob/master/runtime/utils/string.h
+// Retrieved in Aug 2023.
+
+// Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+//               2023 Jing Du (thuduj12@163.com)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef UTILS_UTF8_STRING_H_
+#define UTILS_UTF8_STRING_H_
+
+#include <string>
+#include <vector>
+
+namespace funasr {
+extern const char* WHITESPACE;
+
+int char_length(char ch);
+
+int string_length(const std::string& str);
+
+void string2chars(const std::string& str, std::vector<std::string>* chars);
+
+std::string ltrim(const std::string& str);
+
+std::string rtrim(const std::string& str);
+
+std::string trim(const std::string& str);
+
+void split_string(const std::string& str, const std::string& delim,
+                  std::vector<std::string>* output);
+
+}  // namespace funasr
+
+#endif  // UTILS_UTF8_STRING_H_