Initial commit: FunASR Speech Recognition Toolkit

Add complete FunASR codebase including models, runtime, and documentation.

funasr/models/fsmn_vad_streaming/dynamic_vad.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+"""DynamicStreamingVAD — 动态阈值流式 VAD 封装。
+
+在 fsmn-vad 基础上，根据当前语音段的累积时长动态调整静音切分阈值：
+短句等待更长静音（避免切碎），长句快速切分（避免堆积）。
+
+支持流式（逐帧喂入）和非流式（一次性处理完整音频）两种调用方式。
+
+Usage (流式):
+    from funasr import AutoModel
+    from funasr.models.fsmn_vad_streaming.dynamic_vad import DynamicStreamingVAD
+
+    vad_model = AutoModel(model="fsmn-vad", device="cuda:0")
+    vad = DynamicStreamingVAD(vad_model)
+
+    for audio_chunk in audio_stream:
+        segments = vad.feed(audio_chunk)
+        for seg in segments:
+            print(f"Speech: {seg[0]}-{seg[1]}ms")
+
+    # 结束时
+    final_segments = vad.finalize()
+
+Usage (非流式):
+    segments = vad.process(full_audio_tensor)
+    for seg in segments:
+        print(f"Speech: {seg[0]}-{seg[1]}ms")
+"""
+
+from typing import List, Optional, Tuple
+
+import torch
+import numpy as np
+
+
+# 默认动态阈值配置：(累积时长上限ms, 静音阈值ms)
+DEFAULT_SILENCE_SCHEDULE = [
+    (5000, 2000),
+    (10000, 1500),
+    (15000, 1000),
+    (30000, 800),
+    (45000, 400),
+    (float('inf'), 100),
+]
+
+
+class DynamicStreamingVAD:
+    """动态阈值流式 VAD。
+
+    在 fsmn-vad 的流式推理基础上，根据当前语音段已累积的时长
+    动态调整静音切分阈值，实现「短句不切碎、长句快切分」。
+
+    Args:
+        vad_model: FunASR AutoModel 加载的 fsmn-vad 模型实例。
+        chunk_size_ms: 每次喂入 VAD 的 chunk 大小（毫秒），默认 60。
+        speech_noise_thres: 语音/噪声判别阈值，默认 0.5。
+        speech_to_sil_thres_ms: 语音转静音的基础时间（毫秒），默认 150。
+        silence_schedule: 动态阈值配置表，格式为
+            [(累积时长上限ms, 对应的静音阈值ms), ...]。
+            当累积时长 <= 上限时，使用对应的静音阈值。
+            默认值适合实时对话场景。设为 None 禁用动态调整（使用固定阈值）。
+        sample_rate: 采样率，默认 16000。
+
+    Example:
+        # 自定义阈值：更激进的切分
+        vad = DynamicStreamingVAD(
+            vad_model,
+            silence_schedule=[
+                (3000, 1500),
+                (8000, 800),
+                (15000, 400),
+                (float('inf'), 200),
+            ],
+        )
+    """
+
+    def __init__(
+        self,
+        vad_model,
+        chunk_size_ms: int = 60,
+        speech_noise_thres: float = 0.5,
+        speech_to_sil_thres_ms: int = 150,
+        silence_schedule: Optional[List[Tuple[float, int]]] = None,
+        sample_rate: int = 16000,
+    ):
+        self.model = vad_model
+        self.chunk_size_ms = chunk_size_ms
+        self.speech_noise_thres = speech_noise_thres
+        self.speech_to_sil_thres_ms = speech_to_sil_thres_ms
+        self.silence_schedule = silence_schedule if silence_schedule is not None else DEFAULT_SILENCE_SCHEDULE
+        self.sample_rate = sample_rate
+
+        self.cache = {}
+        self.confirmed_segments: List[List[int]] = []
+        self.current_speech_start: Optional[int] = None
+        self.accumulated_since_cut_ms: int = 0
+
+    def _get_silence_threshold(self) -> int:
+        """根据当前累积时长，从 schedule 中查询静音阈值。"""
+        for limit_ms, silence_ms in self.silence_schedule:
+            if self.accumulated_since_cut_ms <= limit_ms:
+                return silence_ms
+        return self.silence_schedule[-1][1]
+
+    def _apply_dynamic_threshold(self):
+        """将动态阈值应用到 VAD 内部 cache。"""
+        if "stats" not in self.cache:
+            return
+        stats = self.cache["stats"]
+        stats.speech_noise_thres = self.speech_noise_thres
+        desired_silence_ms = self._get_silence_threshold()
+        stats.max_end_sil_frame_cnt_thresh = max(desired_silence_ms - self.speech_to_sil_thres_ms, 0)
+
+    def feed(self, audio_chunk: torch.Tensor, is_final: bool = False) -> List[List[int]]:
+        """喂入一段音频，返回新确认的语音段。
+
+        Args:
+            audio_chunk: 音频数据（float32 tensor，16kHz）。
+                可以是任意长度，内部按 chunk_size_ms 处理。
+            is_final: 是否为最后一段音频。设为 True 时会强制结束当前语音段。
+
+        Returns:
+            新确认的语音段列表，每段为 [start_ms, end_ms]。
+            仅在检测到语音结束时返回非空列表。
+        """
+        if audio_chunk.dim() > 1:
+            audio_chunk = audio_chunk.squeeze()
+
+        chunk_samples = len(audio_chunk)
+        self.accumulated_since_cut_ms += int(chunk_samples * 1000 / self.sample_rate)
+
+        self._apply_dynamic_threshold()
+
+        res = self.model.generate(
+            input=[audio_chunk], cache=self.cache,
+            is_final=is_final, chunk_size=self.chunk_size_ms,
+        )
+
+        signals = res[0].get("value", [])
+        new_confirmed = []
+
+        for sig in signals:
+            if sig[0] >= 0 and sig[1] == -1:
+                self.current_speech_start = sig[0]
+            elif sig[0] == -1 and sig[1] >= 0:
+                start = self.current_speech_start if self.current_speech_start is not None else 0
+                seg = [start, sig[1]]
+                self.confirmed_segments.append(seg)
+                new_confirmed.append(seg)
+                self.current_speech_start = None
+                self.accumulated_since_cut_ms = 0
+            elif sig[0] >= 0 and sig[1] >= 0:
+                self.confirmed_segments.append(sig)
+                new_confirmed.append(sig)
+                self.current_speech_start = None
+                self.accumulated_since_cut_ms = 0
+
+        return new_confirmed
+
+    def finalize(self) -> List[List[int]]:
+        """结束流式处理，返回最后可能未结束的语音段。
+
+        调用此方法后，VAD 状态会被重置。
+        如果当前有正在进行的语音段，会被强制结束。
+
+        Returns:
+            最后确认的语音段列表。
+        """
+        # Feed empty with is_final=True to flush
+        empty = torch.zeros(int(self.sample_rate * 0.01), dtype=torch.float32)
+        return self.feed(empty, is_final=True)
+
+    def process(self, audio: torch.Tensor) -> List[List[int]]:
+        """非流式接口：一次性处理完整音频，返回所有语音段。
+
+        Args:
+            audio: 完整音频（float32 tensor，16kHz）。
+
+        Returns:
+            所有检测到的语音段 [[start_ms, end_ms], ...]。
+        """
+        self.reset()
+
+        if isinstance(audio, np.ndarray):
+            audio = torch.from_numpy(audio).float()
+        if audio.dim() > 1:
+            audio = audio.squeeze()
+
+        # 分 chunk 喂入
+        chunk_samples = int(self.sample_rate * self.chunk_size_ms / 1000)
+        total = len(audio)
+        all_segments = []
+
+        for i in range(0, total, chunk_samples):
+            chunk = audio[i:i + chunk_samples]
+            is_last = (i + chunk_samples >= total)
+            segs = self.feed(chunk, is_final=is_last)
+            all_segments.extend(segs)
+
+        return all_segments
+
+    @property
+    def is_speaking(self) -> bool:
+        """当前是否在语音状态中。"""
+        return self.current_speech_start is not None
+
+    @property
+    def current_duration_ms(self) -> int:
+        """当前段已累积的时长（毫秒）。"""
+        return self.accumulated_since_cut_ms
+
+    @property
+    def current_threshold_ms(self) -> int:
+        """当前使用的静音阈值（毫秒）。"""
+        return self._get_silence_threshold()
+
+    def reset(self):
+        """重置所有状态，开始新一轮检测。"""
+        self.cache = {}
+        self.confirmed_segments = []
+        self.current_speech_start = None
+        self.accumulated_since_cut_ms = 0

funasr/models/fsmn_vad_streaming/encoder.py

@@ -0,0 +1,453 @@
+from typing import Tuple, Dict
+import copy
+import os
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from funasr.register import tables
+
+
+class LinearTransform(nn.Module):
+
+    def __init__(self, input_dim, output_dim):
+        """Initialize LinearTransform.
+        
+            Args:
+                input_dim: Size/dimension parameter.
+                output_dim: Size/dimension parameter.
+            """
+        super(LinearTransform, self).__init__()
+        self.input_dim = input_dim
+        self.output_dim = output_dim
+        self.linear = nn.Linear(input_dim, output_dim, bias=False)
+
+    def forward(self, input):
+        """Forward pass for training.
+        
+            Args:
+                input: Input audio/text data.
+            """
+        output = self.linear(input)
+
+        return output
+
+
+class AffineTransform(nn.Module):
+
+    def __init__(self, input_dim, output_dim):
+        """Initialize AffineTransform.
+        
+            Args:
+                input_dim: Size/dimension parameter.
+                output_dim: Size/dimension parameter.
+            """
+        super(AffineTransform, self).__init__()
+        self.input_dim = input_dim
+        self.output_dim = output_dim
+        self.linear = nn.Linear(input_dim, output_dim)
+
+    def forward(self, input):
+        """Forward pass for training.
+        
+            Args:
+                input: Input audio/text data.
+            """
+        output = self.linear(input)
+
+        return output
+
+
+class RectifiedLinear(nn.Module):
+
+    def __init__(self, input_dim, output_dim):
+        """Initialize RectifiedLinear.
+        
+            Args:
+                input_dim: Size/dimension parameter.
+                output_dim: Size/dimension parameter.
+            """
+        super(RectifiedLinear, self).__init__()
+        self.dim = input_dim
+        self.relu = nn.ReLU()
+        self.dropout = nn.Dropout(0.1)
+
+    def forward(self, input):
+        """Forward pass for training.
+        
+            Args:
+                input: Input audio/text data.
+            """
+        out = self.relu(input)
+        return out
+
+
+class FSMNBlock(nn.Module):
+
+    def __init__(
+        self,
+        input_dim: int,
+        output_dim: int,
+        lorder=None,
+        rorder=None,
+        lstride=1,
+        rstride=1,
+    ):
+        """Initialize FSMNBlock.
+        
+            Args:
+                input_dim: Size/dimension parameter.
+                output_dim: Size/dimension parameter.
+                lorder: TODO.
+                rorder: TODO.
+                lstride: TODO.
+                rstride: TODO.
+            """
+        super(FSMNBlock, self).__init__()
+
+        self.dim = input_dim
+
+        if lorder is None:
+            return
+
+        self.lorder = lorder
+        self.rorder = rorder
+        self.lstride = lstride
+        self.rstride = rstride
+
+        self.conv_left = nn.Conv2d(
+            self.dim, self.dim, [lorder, 1], dilation=[lstride, 1], groups=self.dim, bias=False
+        )
+
+        if self.rorder > 0:
+            self.conv_right = nn.Conv2d(
+                self.dim, self.dim, [rorder, 1], dilation=[rstride, 1], groups=self.dim, bias=False
+            )
+        else:
+            self.conv_right = None
+
+    def forward(self, input: torch.Tensor, cache: torch.Tensor = None):
+        """Forward pass for training.
+        
+            Args:
+                input: Input audio/text data.
+                cache: State cache dict for streaming inference.
+            """
+        x = torch.unsqueeze(input, 1)
+        x_per = x.permute(0, 3, 2, 1)  # B D T C
+
+        if cache is not None:
+            cache = cache.to(x_per.device)
+            y_left = torch.cat((cache, x_per), dim=2)
+            cache = y_left[:, :, -(self.lorder - 1) * self.lstride :, :]
+        else:
+            y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0])
+
+        y_left = self.conv_left(y_left)
+        out = x_per + y_left
+
+        if self.conv_right is not None:
+            # maybe need to check
+            y_right = F.pad(x_per, [0, 0, 0, self.rorder * self.rstride])
+            y_right = y_right[:, :, self.rstride :, :]
+            y_right = self.conv_right(y_right)
+            out += y_right
+
+        out_per = out.permute(0, 3, 2, 1)
+        output = out_per.squeeze(1)
+
+        return output, cache
+
+
+class BasicBlock(nn.Module):
+    def __init__(
+        self,
+        linear_dim: int,
+        proj_dim: int,
+        lorder: int,
+        rorder: int,
+        lstride: int,
+        rstride: int,
+        stack_layer: int,
+    ):
+        """Initialize BasicBlock.
+        
+            Args:
+                linear_dim: Size/dimension parameter.
+                proj_dim: Size/dimension parameter.
+                lorder: TODO.
+                rorder: TODO.
+                lstride: TODO.
+                rstride: TODO.
+                stack_layer: TODO.
+            """
+        super(BasicBlock, self).__init__()
+        self.lorder = lorder
+        self.rorder = rorder
+        self.lstride = lstride
+        self.rstride = rstride
+        self.stack_layer = stack_layer
+        self.linear = LinearTransform(linear_dim, proj_dim)
+        self.fsmn_block = FSMNBlock(proj_dim, proj_dim, lorder, rorder, lstride, rstride)
+        self.affine = AffineTransform(proj_dim, linear_dim)
+        self.relu = RectifiedLinear(linear_dim, linear_dim)
+
+    def forward(self, input: torch.Tensor, cache: Dict[str, torch.Tensor] = None):
+        """Forward pass for training.
+        
+            Args:
+                input: Input audio/text data.
+                cache: State cache dict for streaming inference.
+            """
+        x1 = self.linear(input)  # B T D
+
+        if cache is not None:
+            cache_layer_name = 'cache_layer_{}'.format(self.stack_layer)
+            if cache_layer_name not in cache:
+                cache[cache_layer_name] = torch.zeros(
+                    x1.shape[0], x1.shape[-1], (self.lorder - 1) * self.lstride, 1
+                )
+            x2, cache[cache_layer_name] = self.fsmn_block(x1, cache[cache_layer_name])
+        else:
+            x2, _ = self.fsmn_block(x1, None)
+        x3 = self.affine(x2)
+        x4 = self.relu(x3)
+        return x4
+
+
+class BasicBlock_export(nn.Module):
+    def __init__(
+        self,
+        model,
+    ):
+        """Initialize BasicBlock_export.
+        
+            Args:
+                model: Model instance or model name.
+            """
+        super(BasicBlock_export, self).__init__()
+        self.linear = model.linear
+        self.fsmn_block = model.fsmn_block
+        self.affine = model.affine
+        self.relu = model.relu
+
+    def forward(self, input: torch.Tensor, in_cache: torch.Tensor):
+        """Forward pass for training.
+        
+            Args:
+                input: Input audio/text data.
+                in_cache: TODO.
+            """
+        x = self.linear(input)  # B T D
+        # cache_layer_name = 'cache_layer_{}'.format(self.stack_layer)
+        # if cache_layer_name not in in_cache:
+        #     in_cache[cache_layer_name] = torch.zeros(x1.shape[0], x1.shape[-1], (self.lorder - 1) * self.lstride, 1)
+        x, out_cache = self.fsmn_block(x, in_cache)
+        x = self.affine(x)
+        x = self.relu(x)
+        return x, out_cache
+
+
+class FsmnStack(nn.Sequential):
+    def __init__(self, *args):
+        """Initialize FsmnStack.
+        
+            Args:
+                *args: Variable positional arguments.
+            """
+        super(FsmnStack, self).__init__(*args)
+
+    def forward(self, input: torch.Tensor, cache: Dict[str, torch.Tensor]):
+        """Forward pass for training.
+        
+            Args:
+                input: Input audio/text data.
+                cache: State cache dict for streaming inference.
+            """
+        x = input
+        for module in self._modules.values():
+            x = module(x, cache)
+        return x
+
+
+"""
+FSMN net for keyword spotting
+input_dim:              input dimension
+linear_dim:             fsmn input dimensionll
+proj_dim:               fsmn projection dimension
+lorder:                 fsmn left order
+rorder:                 fsmn right order
+num_syn:                output dimension
+fsmn_layers:            no. of sequential fsmn layers
+"""
+
+
+@tables.register("encoder_classes", "FSMN")
+class FSMN(nn.Module):
+    def __init__(
+        self,
+        input_dim: int,
+        input_affine_dim: int,
+        fsmn_layers: int,
+        linear_dim: int,
+        proj_dim: int,
+        lorder: int,
+        rorder: int,
+        lstride: int,
+        rstride: int,
+        output_affine_dim: int,
+        output_dim: int,
+        use_softmax: bool = True,
+    ):
+        """Initialize FSMN.
+        
+            Args:
+                input_dim: Size/dimension parameter.
+                input_affine_dim: Size/dimension parameter.
+                fsmn_layers: TODO.
+                linear_dim: Size/dimension parameter.
+                proj_dim: Size/dimension parameter.
+                lorder: TODO.
+                rorder: TODO.
+                lstride: TODO.
+                rstride: TODO.
+                output_affine_dim: Size/dimension parameter.
+                output_dim: Size/dimension parameter.
+                use_softmax: TODO.
+            """
+        super().__init__()
+
+        self.input_dim = input_dim
+        self.input_affine_dim = input_affine_dim
+        self.fsmn_layers = fsmn_layers
+        self.linear_dim = linear_dim
+        self.proj_dim = proj_dim
+        self.output_affine_dim = output_affine_dim
+        self.output_dim = output_dim
+
+        self.in_linear1 = AffineTransform(input_dim, input_affine_dim)
+        self.in_linear2 = AffineTransform(input_affine_dim, linear_dim)
+        self.relu = RectifiedLinear(linear_dim, linear_dim)
+        self.fsmn = FsmnStack(
+            *[
+                BasicBlock(linear_dim, proj_dim, lorder, rorder, lstride, rstride, i)
+                for i in range(fsmn_layers)
+            ]
+        )
+        self.out_linear1 = AffineTransform(linear_dim, output_affine_dim)
+        self.out_linear2 = AffineTransform(output_affine_dim, output_dim)
+
+        self.use_softmax = use_softmax
+        if self.use_softmax:
+            self.softmax = nn.Softmax(dim=-1)
+
+    def fuse_modules(self):
+        """Fuse modules."""
+        pass
+
+    def output_size(self) -> int:
+        """Output size."""
+        return self.output_dim
+
+    def forward(
+        self,
+        input: torch.Tensor,
+        cache: Dict[str, torch.Tensor] = None
+    ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]:
+        """
+        Args:
+            input (torch.Tensor): Input tensor (B, T, D)
+            cache: when cache is not None, the forward is in streaming. The type of cache is a dict, egs,
+            {'cache_layer_1': torch.Tensor(B, T1, D)}, T1 is equal to self.lorder. It is {} for the 1st frame
+        """
+
+        x1 = self.in_linear1(input)
+        x2 = self.in_linear2(x1)
+        x3 = self.relu(x2)
+        x4 = self.fsmn(x3, cache)  # self.cache will update automatically in self.fsmn
+        x5 = self.out_linear1(x4)
+        x6 = self.out_linear2(x5)
+
+        if self.use_softmax:
+            x7 = self.softmax(x6)
+            return x7
+
+        return x6
+
+
+@tables.register("encoder_classes", "FSMNExport")
+class FSMNExport(nn.Module):
+    def __init__(
+        self,
+        model,
+        **kwargs,
+    ):
+        """Initialize FSMNExport.
+        
+            Args:
+                model: Model instance or model name.
+                **kwargs: Additional keyword arguments.
+            """
+        super().__init__()
+
+        # self.input_dim = input_dim
+        # self.input_affine_dim = input_affine_dim
+        # self.fsmn_layers = fsmn_layers
+        # self.linear_dim = linear_dim
+        # self.proj_dim = proj_dim
+        # self.output_affine_dim = output_affine_dim
+        # self.output_dim = output_dim
+        #
+        # self.in_linear1 = AffineTransform(input_dim, input_affine_dim)
+        # self.in_linear2 = AffineTransform(input_affine_dim, linear_dim)
+        # self.relu = RectifiedLinear(linear_dim, linear_dim)
+        # self.fsmn = FsmnStack(*[BasicBlock(linear_dim, proj_dim, lorder, rorder, lstride, rstride, i) for i in
+        #                         range(fsmn_layers)])
+        # self.out_linear1 = AffineTransform(linear_dim, output_affine_dim)
+        # self.out_linear2 = AffineTransform(output_affine_dim, output_dim)
+        # self.softmax = nn.Softmax(dim=-1)
+
+        self.in_linear1 = model.in_linear1
+        self.in_linear2 = model.in_linear2
+        self.relu = model.relu
+        # self.fsmn = model.fsmn
+        self.out_linear1 = model.out_linear1
+        self.out_linear2 = model.out_linear2
+        self.softmax = model.softmax
+        self.fsmn = model.fsmn
+        for i, d in enumerate(model.fsmn):
+            if isinstance(d, BasicBlock):
+                self.fsmn[i] = BasicBlock_export(d)
+
+    def fuse_modules(self):
+        """Fuse modules."""
+        pass
+
+    def forward(
+        self,
+        input: torch.Tensor,
+        *args,
+    ):
+        """
+        Args:
+            input (torch.Tensor): Input tensor (B, T, D)
+            in_cache: when in_cache is not None, the forward is in streaming. The type of in_cache is a dict, egs,
+            {'cache_layer_1': torch.Tensor(B, T1, D)}, T1 is equal to self.lorder. It is {} for the 1st frame
+        """
+
+        x = self.in_linear1(input)
+        x = self.in_linear2(x)
+        x = self.relu(x)
+        # x4 = self.fsmn(x3, in_cache)  # self.in_cache will update automatically in self.fsmn
+        out_caches = list()
+        for i, d in enumerate(self.fsmn):
+            in_cache = args[i]
+            x, out_cache = d(x, in_cache)
+            out_caches.append(out_cache)
+        x = self.out_linear1(x)
+        x = self.out_linear2(x)
+        x = self.softmax(x)
+
+        return x, out_caches

funasr/models/fsmn_vad_streaming/export_meta.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+import types
+import torch
+from funasr.register import tables
+
+
+def export_rebuild_model(model, **kwargs):
+    """Export rebuild model.
+    
+        Args:
+            model: Model instance or model name.
+            **kwargs: Additional keyword arguments.
+        """
+    is_onnx = kwargs.get("type", "onnx") == "onnx"
+    encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
+    model.encoder = encoder_class(model.encoder, onnx=is_onnx)
+
+    model.forward = types.MethodType(export_forward, model)
+    model.export_dummy_inputs = types.MethodType(export_dummy_inputs, model)
+    model.export_input_names = types.MethodType(export_input_names, model)
+    model.export_output_names = types.MethodType(export_output_names, model)
+    model.export_dynamic_axes = types.MethodType(export_dynamic_axes, model)
+    model.export_name = types.MethodType(export_name, model)
+
+    return model
+
+
+def export_forward(self, feats: torch.Tensor, *args, **kwargs):
+
+    """Export forward.
+    
+        Args:
+            feats: Feature tensor (e.g., fbank), shape (batch, frames, dim).
+            *args: Variable positional arguments.
+            **kwargs: Additional keyword arguments.
+        """
+    scores, out_caches = self.encoder(feats, *args)
+
+    return scores, out_caches
+
+
+def export_dummy_inputs(self, data_in=None, frame=30):
+    """Export dummy inputs.
+    
+        Args:
+            data_in: Input data (audio samples, file paths, or text).
+            frame: TODO.
+        """
+    if data_in is None:
+        speech = torch.randn(1, frame, self.encoder_conf.get("input_dim"))
+    else:
+        speech = None  # Undo
+
+    cache_frames = self.encoder_conf.get("lorder") + self.encoder_conf.get("rorder") - 1
+    in_cache0 = torch.randn(1, self.encoder_conf.get("proj_dim"), cache_frames, 1)
+    in_cache1 = torch.randn(1, self.encoder_conf.get("proj_dim"), cache_frames, 1)
+    in_cache2 = torch.randn(1, self.encoder_conf.get("proj_dim"), cache_frames, 1)
+    in_cache3 = torch.randn(1, self.encoder_conf.get("proj_dim"), cache_frames, 1)
+
+    return (speech, in_cache0, in_cache1, in_cache2, in_cache3)
+
+
+def export_input_names(self):
+    """Export input names."""
+    return ["speech", "in_cache0", "in_cache1", "in_cache2", "in_cache3"]
+
+
+def export_output_names(self):
+    """Export output names."""
+    return ["logits", "out_cache0", "out_cache1", "out_cache2", "out_cache3"]
+
+
+def export_dynamic_axes(self):
+    """Export dynamic axes."""
+    return {
+        "speech": {1: "feats_length"},
+    }
+
+
+def export_name(
+    self,
+):
+    """Export name."""
+    return "model.onnx"

funasr/models/fsmn_vad_streaming/template.yaml

@@ -0,0 +1,62 @@
+# This is an example that demonstrates how to configure a model file.
+# You can modify the configuration according to your own requirements.
+
+# to print the register_table:
+# from funasr.register import tables
+# tables.print()
+
+# network architecture
+model: FsmnVADStreaming
+model_conf:
+    sample_rate: 16000
+    detect_mode: 1 
+    snr_mode: 0
+    max_end_silence_time: 800
+    max_start_silence_time: 3000
+    do_start_point_detection: True
+    do_end_point_detection: True
+    window_size_ms: 200
+    sil_to_speech_time_thres: 150
+    speech_to_sil_time_thres: 150
+    speech_2_noise_ratio: 1.0
+    do_extend: 1
+    lookback_time_start_point: 200
+    lookahead_time_end_point: 100
+    max_single_segment_time: 60000
+    snr_thres: -100.0
+    noise_frame_num_used_for_snr: 100
+    decibel_thres: -100.0
+    speech_noise_thres: 0.6
+    fe_prior_thres: 0.0001
+    silence_pdf_num: 1
+    sil_pdf_ids: [0]
+    speech_noise_thresh_low: -0.1
+    speech_noise_thresh_high: 0.3
+    output_frame_probs: False
+    frame_in_ms: 10
+    frame_length_ms: 25
+    
+encoder: FSMN
+encoder_conf:
+    input_dim: 400
+    input_affine_dim: 140
+    fsmn_layers: 4
+    linear_dim: 250
+    proj_dim: 128
+    lorder: 20
+    rorder: 0
+    lstride: 1
+    rstride: 0
+    output_affine_dim: 140
+    output_dim: 248
+
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    dither: 0.0
+    lfr_m: 5
+    lfr_n: 1