Initial commit: FunASR Speech Recognition Toolkit

Add complete FunASR codebase including models, runtime, and documentation.

funasr/models/fun_asr_nano/ctc.py

@@ -0,0 +1,70 @@
+import torch
+import torch.nn.functional as F
+
+
+class CTC(torch.nn.Module):
+    """CTC module.
+
+    Args:
+        odim: dimension of outputs
+        encoder_output_size: number of encoder projection units
+        dropout_rate: dropout rate (0.0 ~ 1.0)
+        reduce: reduce the CTC loss into a scalar
+    """
+
+    def __init__(
+        self,
+        odim: int,
+        encoder_output_size: int,
+        dropout_rate: float = 0.0,
+        reduce: bool = True,
+        blank_id: int = 0,
+        **kwargs,
+    ):
+        """Initialize CTC.
+        
+            Args:
+                odim: TODO.
+                encoder_output_size: Size/dimension parameter.
+                dropout_rate: TODO.
+                reduce: TODO.
+                blank_id: TODO.
+                **kwargs: Additional keyword arguments.
+            """
+        super().__init__()
+        eprojs = encoder_output_size
+        self.dropout_rate = dropout_rate
+        self.ctc_lo = torch.nn.Linear(eprojs, odim)
+        self.blank_id = blank_id
+        self.ctc_loss = torch.nn.CTCLoss(reduction="none", blank=blank_id)
+        self.reduce = reduce
+
+    def softmax(self, hs_pad):
+        """softmax of frame activations
+
+        Args:
+            Tensor hs_pad: 3d tensor (B, Tmax, eprojs)
+        Returns:
+            torch.Tensor: softmax applied 3d tensor (B, Tmax, odim)
+        """
+        return F.softmax(self.ctc_lo(hs_pad), dim=2)
+
+    def log_softmax(self, hs_pad):
+        """log_softmax of frame activations
+
+        Args:
+            Tensor hs_pad: 3d tensor (B, Tmax, eprojs)
+        Returns:
+            torch.Tensor: log softmax applied 3d tensor (B, Tmax, odim)
+        """
+        return F.log_softmax(self.ctc_lo(hs_pad), dim=2)
+
+    def argmax(self, hs_pad):
+        """argmax of frame activations
+
+        Args:
+            torch.Tensor hs_pad: 3d tensor (B, Tmax, eprojs)
+        Returns:
+            torch.Tensor: argmax applied 2d tensor (B, Tmax)
+        """
+        return torch.argmax(self.ctc_lo(hs_pad), dim=2)

funasr/models/fun_asr_nano/device_utils.py

@@ -0,0 +1,34 @@
+# Copyright FunASR (https://github.com/modelscope/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+"""Device helpers for Fun-ASR-Nano runtime paths."""
+
+_SUPPORTED_AUTOCAST_DEVICE_TYPES = {"cuda", "xpu", "mps", "npu"}
+
+
+def _device_type_from_value(device):
+    """Resolve a device type without requiring optional backend registration."""
+    if device is None:
+        return "cpu"
+
+    device_type = getattr(device, "type", None)
+    if device_type:
+        return str(device_type).lower()
+
+    if isinstance(device, str):
+        return device.split(":", 1)[0].lower()
+
+    return str(device).split(":", 1)[0].lower()
+
+
+def resolve_autocast_device_type(device):
+    """Return the torch.autocast device_type for a Fun-ASR-Nano device.
+
+    PyTorch builds without torch_npu may reject ``torch.device("npu:0")`` before
+    torch_npu registers the backend. Parse strings directly so NPU requests do
+    not fall back to CPU autocast, which only supports bf16 and caused #3034.
+    """
+    device_type = _device_type_from_value(device)
+    if device_type in _SUPPORTED_AUTOCAST_DEVICE_TYPES:
+        return device_type
+    return "cpu"

funasr/models/fun_asr_nano/inference_vllm.py

@@ -0,0 +1,728 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+"""
+Fun-ASR-Nano vLLM inference engine.
+
+Uses vLLM for high-throughput LLM decoding while keeping the audio encoder
+and adaptor in PyTorch. Supports batch inference and tensor-parallel for
+multi-GPU acceleration.
+
+Usage:
+    from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+
+    engine = FunASRNanoVLLM.from_pretrained(
+        model="FunAudioLLM/Fun-ASR-Nano-2512",
+        tensor_parallel_size=2,
+    )
+    results = engine.generate(["audio1.wav", "audio2.wav"])
+"""
+
+import glob
+import json
+import logging
+import os
+import re
+import shutil
+import time
+from typing import List, Optional, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+logger = logging.getLogger(__name__)
+
+dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}
+
+
+def prepare_vllm_model_dir(model_dir: str, output_dir: str = None) -> str:
+    """Extract LLM weights from Fun-ASR-Nano model.pt and save in HuggingFace format.
+
+    Fun-ASR-Nano stores all weights (audio encoder + adaptor + LLM) in a single
+    model.pt file. vLLM needs the LLM weights in standard HuggingFace format.
+    This function extracts LLM weights and saves them alongside the config/tokenizer
+    files from the Qwen3-0.6B subdirectory.
+
+    Args:
+        model_dir: Path to the Fun-ASR-Nano model directory.
+        output_dir: Where to save the extracted LLM. Defaults to model_dir/Qwen3-0.6B-vllm.
+
+    Returns:
+        Path to the directory containing the vLLM-ready LLM model.
+    """
+    if output_dir is None:
+        output_dir = os.path.join(model_dir, "Qwen3-0.6B-vllm")
+
+    # Check if already prepared
+    safetensors_files = glob.glob(os.path.join(output_dir, "*.safetensors"))
+    bin_files = glob.glob(os.path.join(output_dir, "model*.bin"))
+    if safetensors_files or bin_files:
+        logger.info(f"vLLM model already prepared at {output_dir}")
+        return output_dir
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Copy config and tokenizer from Qwen3-0.6B
+    qwen_dir = os.path.join(model_dir, "Qwen3-0.6B")
+    if not os.path.isdir(qwen_dir):
+        raise FileNotFoundError(f"Qwen3-0.6B config directory not found at {qwen_dir}")
+
+    for fname in os.listdir(qwen_dir):
+        src = os.path.join(qwen_dir, fname)
+        dst = os.path.join(output_dir, fname)
+        if os.path.isfile(src) and not os.path.exists(dst):
+            shutil.copy2(src, dst)
+
+    # Load model.pt and extract LLM weights
+    model_pt = os.path.join(model_dir, "model.pt")
+    if not os.path.exists(model_pt):
+        raise FileNotFoundError(
+            f"model.pt not found at {model_pt}. Make sure the model is fully downloaded."
+        )
+
+    logger.info(f"Loading model.pt from {model_pt}...")
+    checkpoint = torch.load(model_pt, map_location="cpu")
+    if "state_dict" in checkpoint:
+        state_dict = checkpoint["state_dict"]
+    else:
+        state_dict = checkpoint
+
+    # Extract LLM weights (prefixed with "llm.")
+    llm_state = {}
+    for key, value in state_dict.items():
+        if key.startswith("llm."):
+            new_key = key[len("llm."):]
+            llm_state[new_key] = value
+
+    if not llm_state:
+        raise RuntimeError("No LLM weights found in model.pt (expected prefix 'llm.')")
+
+    logger.info(f"Extracted {len(llm_state)} LLM weight tensors")
+
+    # Save in safetensors format (preferred by vLLM)
+    try:
+        from safetensors.torch import save_file
+
+        save_path = os.path.join(output_dir, "model.safetensors")
+        save_file(llm_state, save_path)
+        logger.info(f"Saved LLM weights to {save_path}")
+
+        # Create model index
+        index = {
+            "metadata": {"total_size": sum(v.numel() * v.element_size() for v in llm_state.values())},
+            "weight_map": {k: "model.safetensors" for k in llm_state.keys()},
+        }
+        with open(os.path.join(output_dir, "model.safetensors.index.json"), "w") as f:
+            json.dump(index, f, indent=2)
+    except ImportError:
+        save_path = os.path.join(output_dir, "model.bin")
+        torch.save(llm_state, save_path)
+        logger.info(f"Saved LLM weights to {save_path} (install safetensors for faster loading)")
+
+    return output_dir
+
+
+class FunASRNanoVLLM:
+    """Fun-ASR-Nano with vLLM backend for high-throughput inference.
+
+    Architecture:
+        Audio -> WavFrontend -> SenseVoiceEncoder -> AudioAdaptor -> audio embeddings
+        Text tokens -> LLM embedding layer -> text embeddings
+        Combined embeddings -> vLLM (Qwen3-0.6B) -> generated text
+
+    The audio encoder and adaptor run in PyTorch on a single GPU,
+    while vLLM handles the LLM inference with optional tensor parallelism.
+
+    Args:
+        model_dir: Path to the Fun-ASR-Nano model directory.
+        device: Device for audio encoder/adaptor (e.g. "cuda:0").
+        dtype: Dtype for audio processing ("bf16", "fp16", "fp32").
+        tensor_parallel_size: Number of GPUs for vLLM tensor parallelism.
+        gpu_memory_utilization: Fraction of GPU memory for vLLM KV cache.
+        max_model_len: Maximum sequence length for vLLM.
+        enforce_eager: Disable CUDA graph for debugging.
+
+    Example:
+        >>> engine = FunASRNanoVLLM(
+        ...     model_dir="/path/to/Fun-ASR-Nano-2512",
+        ...     tensor_parallel_size=2,
+        ... )
+        >>> results = engine.generate(["audio1.wav", "audio2.wav"])
+        >>> for r in results:
+        ...     print(r["text"])
+    """
+
+    def __init__(
+        self,
+        model_dir: str,
+        device: str = "cuda:0",
+        dtype: str = "bf16",
+        tensor_parallel_size: int = 1,
+        gpu_memory_utilization: float = 0.8,
+        max_model_len: int = 2048,
+        enforce_eager: bool = False,
+        **kwargs,
+    ):
+        from vllm import LLM, SamplingParams
+        try:
+            from vllm.inputs import EmbedsPrompt
+        except ImportError:
+            from vllm.inputs.data import EmbedsPrompt
+
+        self.device = device
+        self.dtype = dtype
+        self.torch_dtype = dtype_map.get(dtype, torch.bfloat16)
+        if self.torch_dtype == torch.float16:
+            logger.warning(
+                "dtype='fp16' can produce degraded or garbage transcription for "
+                "Fun-ASR-Nano (numerical overflow in the audio embedding path). "
+                "Use dtype='bf16' (recommended) or dtype='fp32'. On GPUs without "
+                "bfloat16 support (e.g. NVIDIA V100), use 'fp32'."
+            )
+        self.model_dir = model_dir
+
+        # Step 1: Prepare LLM weights for vLLM (extract from model.pt if needed)
+        vllm_model_dir = prepare_vllm_model_dir(model_dir)
+
+        # Step 2: Load audio components (encoder + adaptor + frontend)
+        self._load_audio_components(model_dir, **kwargs)
+
+        # Step 3: Initialize vLLM engine
+        logger.info(f"Initializing vLLM with model: {vllm_model_dir}")
+        logger.info(f"  tensor_parallel_size={tensor_parallel_size}")
+        logger.info(f"  gpu_memory_utilization={gpu_memory_utilization}")
+
+        vllm_kwargs = kwargs.get("vllm_kwargs", {})
+        self.vllm_engine = LLM(
+            enable_prompt_embeds=True,
+            model=vllm_model_dir,
+            tensor_parallel_size=tensor_parallel_size,
+            gpu_memory_utilization=gpu_memory_utilization,
+            max_model_len=max_model_len,
+            enforce_eager=enforce_eager,
+            dtype={"bf16": "bfloat16", "fp16": "float16", "fp32": "auto"}.get(dtype, dtype),
+            trust_remote_code=True,
+            **vllm_kwargs,
+        )
+
+        # Step 4: Get tokenizer and LLM embedding layer
+        self.tokenizer = self.vllm_engine.get_tokenizer()
+        self._load_embedding_layer(model_dir)
+
+    def _load_audio_components(self, model_dir: str, **kwargs):
+        """Load audio encoder, adaptor, frontend, and CTC from checkpoint."""
+        from omegaconf import OmegaConf
+        from funasr.register import tables
+
+        config_path = os.path.join(model_dir, "config.yaml")
+        config = OmegaConf.load(config_path)
+        self._config = OmegaConf.to_container(config, resolve=True)
+
+        # --- Frontend ---
+        frontend_class = tables.frontend_classes.get(config["frontend"])
+        frontend_conf = OmegaConf.to_container(config.get("frontend_conf", {}), resolve=True)
+        cmvn_file = frontend_conf.get("cmvn_file")
+        if cmvn_file and not os.path.isabs(cmvn_file):
+            frontend_conf["cmvn_file"] = os.path.join(model_dir, cmvn_file)
+        self.frontend = frontend_class(**frontend_conf)
+        self.frontend.eval()
+
+        # --- Audio Encoder ---
+        encoder_conf = OmegaConf.to_container(config.get("audio_encoder_conf", {}), resolve=True)
+        hub = encoder_conf.get("hub", None)
+        if hub == "ms":
+            from funasr import AutoModel as FunAutoModel
+
+            enc_model = FunAutoModel(
+                model=config["audio_encoder"], model_revision="master", disable_update=True
+            )
+            self.audio_encoder_output_size = (
+                enc_model.model.encoder_output_size
+                if hasattr(enc_model.model, "encoder_output_size")
+                else -1
+            )
+            self.audio_encoder = (
+                enc_model.model.model.encoder
+                if hasattr(enc_model.model, "model")
+                else enc_model.model.encoder
+            )
+        else:
+            encoder_class = tables.encoder_classes.get(config["audio_encoder"])
+            input_size = self.frontend.output_size()
+            self.audio_encoder = encoder_class(input_size=input_size, **encoder_conf)
+            self.audio_encoder_output_size = self.audio_encoder.output_size()
+
+        self.audio_encoder.eval()
+        for p in self.audio_encoder.parameters():
+            p.requires_grad = False
+
+        # --- Audio Adaptor ---
+        adaptor_conf = OmegaConf.to_container(config.get("audio_adaptor_conf", {}), resolve=True)
+        adaptor_class = tables.adaptor_classes.get(config["audio_adaptor"])
+        if self.audio_encoder_output_size > 0:
+            adaptor_conf["encoder_dim"] = self.audio_encoder_output_size
+        self.audio_adaptor = adaptor_class(**adaptor_conf)
+        self.audio_adaptor.eval()
+        for p in self.audio_adaptor.parameters():
+            p.requires_grad = False
+        self.use_low_frame_rate = adaptor_conf.get("use_low_frame_rate", False)
+
+        # --- CTC Decoder (optional, for timestamps) ---
+        self.ctc_decoder = None
+        self.ctc = None
+        self.ctc_tokenizer = None
+        self.blank_id = None
+
+        ctc_decoder_name = self._config.get("ctc_decoder", None)
+        if ctc_decoder_name:
+            ctc_decoder_class = tables.adaptor_classes.get(ctc_decoder_name)
+            ctc_decoder_conf = self._config.get("ctc_decoder_conf", {})
+            if self.audio_encoder_output_size > 0:
+                ctc_decoder_conf["encoder_dim"] = self.audio_encoder_output_size
+            self.ctc_decoder = ctc_decoder_class(**ctc_decoder_conf)
+            self.ctc_decoder.eval()
+            for p in self.ctc_decoder.parameters():
+                p.requires_grad = False
+
+            from funasr.models.fun_asr_nano.ctc import CTC
+
+            ctc_conf = self._config.get("ctc_conf", {})
+            ctc_vocab_size = self._config.get("ctc_vocab_size", 60515)
+            self.blank_id = ctc_conf.get("blank_id", ctc_vocab_size - 1)
+            self.ctc = CTC(
+                odim=ctc_vocab_size,
+                encoder_output_size=self.audio_encoder_output_size,
+                blank_id=self.blank_id,
+                **ctc_conf,
+            )
+
+            # CTC tokenizer
+            ds_conf = self._config.get("dataset_conf", {})
+            ctc_tokenizer_name = ds_conf.get("ctc_tokenizer", None)
+            ctc_tokenizer_conf = ds_conf.get("ctc_tokenizer_conf", {})
+            if ctc_tokenizer_name:
+                ctc_tokenizer_class = tables.tokenizer_classes.get(ctc_tokenizer_name)
+                vocab_path = ctc_tokenizer_conf.get("vocab_path")
+                if vocab_path is None or not os.path.isabs(vocab_path):
+                    multilingual_path = os.path.join(model_dir, "multilingual.tiktoken")
+                    if os.path.exists(multilingual_path):
+                        ctc_tokenizer_conf["vocab_path"] = multilingual_path
+                    elif vocab_path and not os.path.isabs(vocab_path):
+                        ctc_tokenizer_conf["vocab_path"] = os.path.join(model_dir, vocab_path)
+                self.ctc_tokenizer = ctc_tokenizer_class(**ctc_tokenizer_conf)
+
+        # --- Load weights from model.pt ---
+        model_pt = os.path.join(model_dir, "model.pt")
+        if os.path.exists(model_pt):
+            logger.info(f"Loading audio component weights from {model_pt}")
+            checkpoint = torch.load(model_pt, map_location="cpu")
+            state_dict = checkpoint.get("state_dict", checkpoint)
+
+            # Audio encoder
+            enc_state = {
+                k[len("audio_encoder."):]: v
+                for k, v in state_dict.items()
+                if k.startswith("audio_encoder.")
+            }
+            if enc_state:
+                self.audio_encoder.load_state_dict(enc_state, strict=False)
+                logger.info(f"  Loaded audio_encoder: {len(enc_state)} params")
+
+            # Audio adaptor
+            adp_state = {
+                k[len("audio_adaptor."):]: v
+                for k, v in state_dict.items()
+                if k.startswith("audio_adaptor.")
+            }
+            if adp_state:
+                self.audio_adaptor.load_state_dict(adp_state, strict=False)
+                logger.info(f"  Loaded audio_adaptor: {len(adp_state)} params")
+
+            # CTC decoder
+            if self.ctc_decoder is not None:
+                ctc_dec_state = {
+                    k[len("ctc_decoder."):]: v
+                    for k, v in state_dict.items()
+                    if k.startswith("ctc_decoder.")
+                }
+                if ctc_dec_state:
+                    self.ctc_decoder.load_state_dict(ctc_dec_state, strict=False)
+                ctc_state = {
+                    k[len("ctc."):]: v
+                    for k, v in state_dict.items()
+                    if k.startswith("ctc.") and not k.startswith("ctc_decoder.")
+                }
+                if ctc_state:
+                    self.ctc.load_state_dict(ctc_state, strict=False)
+
+        # Move to device
+        self.audio_encoder = self.audio_encoder.to(self.device, dtype=torch.float32)
+        self.audio_adaptor = self.audio_adaptor.to(self.device, dtype=self.torch_dtype)
+        if self.ctc_decoder is not None:
+            self.ctc_decoder = self.ctc_decoder.to(self.device, dtype=torch.float32)
+            self.ctc = self.ctc.to(self.device, dtype=torch.float32)
+
+    def _load_embedding_layer(self, model_dir: str):
+        """Load the LLM embedding layer for text token embedding computation."""
+        model_pt = os.path.join(model_dir, "model.pt")
+        checkpoint = torch.load(model_pt, map_location="cpu")
+        state_dict = checkpoint.get("state_dict", checkpoint)
+
+        # Look for embedding weights
+        embed_key = None
+        for key in state_dict.keys():
+            if "embed_tokens.weight" in key and key.startswith("llm."):
+                embed_key = key
+                break
+
+        if embed_key is None:
+            raise RuntimeError("Could not find LLM embedding weights in model.pt")
+
+        embed_weight = state_dict[embed_key]
+        self.embed_tokens = nn.Embedding.from_pretrained(embed_weight, freeze=True)
+        self.embed_tokens = self.embed_tokens.to(self.device, dtype=self.torch_dtype)
+        logger.info(f"Loaded embedding layer: {embed_weight.shape}")
+
+    @torch.no_grad()
+    def _encode_audio(self, audio_input: Union[str, torch.Tensor, np.ndarray]):
+        """Encode audio through frontend -> encoder -> adaptor.
+
+        Returns:
+            adaptor_out: (1, T', D_llm) audio embeddings for LLM input
+            adaptor_out_lens: (1,) lengths
+            encoder_out: (1, T, D_enc) encoder output for CTC
+            encoder_out_lens: (1,) encoder output lengths
+        """
+        from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
+
+        if isinstance(audio_input, str):
+            data_src = load_audio_text_image_video(audio_input, fs=self.frontend.fs)
+        elif isinstance(audio_input, np.ndarray):
+            data_src = torch.from_numpy(audio_input).float()
+        elif isinstance(audio_input, torch.Tensor):
+            data_src = audio_input.float()
+        else:
+            raise ValueError(f"Unsupported audio input type: {type(audio_input)}")
+
+        speech, speech_lengths = extract_fbank(
+            data_src, data_type="sound", frontend=self.frontend, is_final=True
+        )
+        speech = speech.to(self.device, dtype=torch.float32)
+        speech_lengths = speech_lengths.to(self.device)
+
+        encoder_out, encoder_out_lens = self.audio_encoder(speech, speech_lengths)
+        encoder_out_for_adaptor = encoder_out.to(dtype=self.torch_dtype)
+        adaptor_out, adaptor_out_lens = self.audio_adaptor(encoder_out_for_adaptor, encoder_out_lens)
+
+        # Apply low frame rate: compute effective token count from fbank length
+        # Matches PyTorch model.py data_load_speech formula exactly
+        if self.use_low_frame_rate:
+            for i in range(adaptor_out.shape[0]):
+                fbank_len = speech_lengths[i].item()
+                olens = 1 + (fbank_len - 3 + 2 * 1) // 2
+                olens = 1 + (olens - 3 + 2 * 1) // 2
+                fake_token_len = (olens - 1) // 2 + 1
+                adaptor_out_lens[i] = fake_token_len
+
+        return adaptor_out, adaptor_out_lens, encoder_out, encoder_out_lens
+
+    def _build_prompt_text(
+        self,
+        hotwords: List[str] = None,
+        language: str = None,
+        itn: bool = True,
+    ) -> str:
+        """Build the ASR prompt string."""
+        hotwords = hotwords or []
+        if len(hotwords) > 0:
+            hotwords_str = ", ".join(hotwords)
+            prompt = (
+                "请结合上下文信息，更加准确地完成语音转写任务。"
+                "如果没有相关信息，我们会留空。\n\n\n**上下文信息：**\n\n\n"
+            )
+            prompt += f"热词列表：[{hotwords_str}]\n"
+        else:
+            prompt = ""
+        if language is None:
+            prompt += "语音转写"
+        else:
+            prompt += f"语音转写成{language}"
+        if not itn:
+            prompt += "，不进行文本规整"
+        return prompt + "："
+
+    @torch.no_grad()
+    def _build_input_embeds(
+        self,
+        audio_embeds: torch.Tensor,
+        audio_embed_lens: torch.Tensor,
+        hotwords: List[str] = None,
+        language: str = None,
+        itn: bool = True,
+        system_prompt: str = "You are a helpful assistant.",
+    ) -> torch.Tensor:
+        """Build the full input embedding sequence with audio inserted.
+
+        Returns:
+            Tensor of shape (seq_len, D_llm)
+        """
+        prompt = self._build_prompt_text(hotwords, language, itn)
+
+        # ChatML format with speech markers and thinking prefix
+        prefix_text = (
+            f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
+            f"<|im_start|>user\n{prompt}<|startofspeech|>"
+        )
+        suffix_text = "<|endofspeech|><|im_end|>\n<|im_start|>assistant\n"
+
+        # Tokenize
+        prefix_ids = self.tokenizer.encode(prefix_text, add_special_tokens=False)
+        suffix_ids = self.tokenizer.encode(suffix_text, add_special_tokens=False)
+
+        # Embed text tokens
+        prefix_tensor = torch.tensor(prefix_ids, dtype=torch.long, device=self.device)
+        suffix_tensor = torch.tensor(suffix_ids, dtype=torch.long, device=self.device)
+        prefix_embeds = self.embed_tokens(prefix_tensor)
+        suffix_embeds = self.embed_tokens(suffix_tensor)
+
+        # Audio embeddings
+        audio_len = audio_embed_lens[0].item()
+        audio_emb = audio_embeds[0, :audio_len, :]
+
+        # Concat: [prefix_text_emb | audio_emb | suffix_text_emb]
+        inputs_embeds = torch.cat([prefix_embeds, audio_emb, suffix_embeds], dim=0)
+        return inputs_embeds
+
+    def generate(
+        self,
+        inputs: Union[str, List[str], np.ndarray, torch.Tensor, List],
+        hotwords: List[str] = None,
+        language: str = None,
+        itn: bool = True,
+        max_new_tokens: int = 512,
+        temperature: float = 0.0,
+        top_p: float = 1.0,
+        top_k: int = -1,
+        repetition_penalty: float = 1.0,
+        **kwargs,
+    ) -> List[dict]:
+        """Run batch ASR inference using vLLM.
+
+        Args:
+            inputs: Audio input(s). Accepts:
+                - str: single file path
+                - List[str]: batch of file paths
+                - np.ndarray / torch.Tensor: raw audio samples (16kHz)
+            hotwords: Keywords to boost recognition accuracy.
+            language: Language hint (e.g. "中文", "英文", "日文").
+            itn: Apply inverse text normalization (default True).
+            max_new_tokens: Maximum tokens to generate per sample.
+            temperature: Sampling temperature (0 = greedy decoding).
+            top_p: Nucleus sampling parameter.
+            top_k: Top-k sampling (-1 = disabled).
+            repetition_penalty: Repetition penalty factor.
+
+        Returns:
+            List of result dicts: [{"key": str, "text": str, "timestamps": [...]}]
+        """
+        from vllm import SamplingParams
+        try:
+            from vllm.inputs import EmbedsPrompt
+        except ImportError:
+            from vllm.inputs.data import EmbedsPrompt
+
+        from funasr.models.fun_asr_nano.vllm_utils import resolve_repetition_penalty
+
+        if isinstance(inputs, (str, np.ndarray, torch.Tensor)):
+            inputs = [inputs]
+
+        sampling_params = SamplingParams(
+            max_tokens=max_new_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            top_k=top_k if top_k > 0 else -1,
+            # Prompt-embeds mode has no token IDs to penalize; see #2948.
+            repetition_penalty=resolve_repetition_penalty(repetition_penalty),
+            skip_special_tokens=True,
+        )
+
+        # Batch encode audio and build embedding prompts
+        prompts = []
+        encoder_outputs = []
+
+        t0 = time.perf_counter()
+
+        # Pre-compute text embeddings (shared across batch)
+        prompt_text = self._build_prompt_text(hotwords, language, itn)
+        prefix_text = f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{prompt_text}"
+        suffix_text = "<|im_end|>\n<|im_start|>assistant\n"
+        prefix_ids = self.tokenizer.encode(prefix_text, add_special_tokens=False)
+        suffix_ids = self.tokenizer.encode(suffix_text, add_special_tokens=False)
+        prefix_emb = self.embed_tokens(torch.tensor(prefix_ids, dtype=torch.long, device=self.device))
+        suffix_emb = self.embed_tokens(torch.tensor(suffix_ids, dtype=torch.long, device=self.device))
+
+        # Batch encode audio (groups of 8 for memory efficiency)
+        batch_size_enc = 8
+        all_adaptor_outs = []
+        all_adaptor_lens = []
+        for i in range(0, len(inputs), batch_size_enc):
+            batch_inputs = inputs[i:i+batch_size_enc]
+            # Load and extract fbank for batch
+            from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
+            audio_tensors = []
+            for audio_input in batch_inputs:
+                if isinstance(audio_input, str):
+                    data_src = load_audio_text_image_video(audio_input, fs=self.frontend.fs)
+                elif isinstance(audio_input, np.ndarray):
+                    data_src = torch.from_numpy(audio_input).float()
+                elif isinstance(audio_input, torch.Tensor):
+                    data_src = audio_input.float()
+                else:
+                    raise ValueError(f"Unsupported audio input type: {type(audio_input)}")
+                audio_tensors.append(data_src)
+
+            speech, speech_lengths = extract_fbank(
+                audio_tensors, data_type="sound", frontend=self.frontend, is_final=True
+            )
+            speech = speech.to(self.device, dtype=torch.float32)
+            speech_lengths = speech_lengths.to(self.device)
+
+            with torch.no_grad():
+                enc_out, enc_lens = self.audio_encoder(speech, speech_lengths)
+                adp_out, adp_lens = self.audio_adaptor(enc_out.to(dtype=self.torch_dtype), enc_lens)
+
+            # Apply low frame rate token length correction
+            if self.use_low_frame_rate:
+                for j in range(len(batch_inputs)):
+                    fbank_len = speech_lengths[j].item()
+                    olens = 1 + (fbank_len - 3 + 2 * 1) // 2
+                    olens = 1 + (olens - 3 + 2 * 1) // 2
+                    adp_lens[j] = (olens - 1) // 2 + 1
+
+            for j in range(len(batch_inputs)):
+                all_adaptor_outs.append(adp_out[j, :adp_lens[j], :])
+                all_adaptor_lens.append(adp_lens[j])
+                encoder_outputs.append((enc_out[j:j+1, :enc_lens[j], :], enc_lens[j:j+1]))
+
+        # Build prompts
+        for audio_emb in all_adaptor_outs:
+            input_embeds = torch.cat([prefix_emb, audio_emb, suffix_emb], dim=0)
+            prompts.append(EmbedsPrompt(prompt_embeds=input_embeds.float()))
+
+        t1 = time.perf_counter()
+        logger.info(f"Audio encoding: {len(inputs)} samples in {t1 - t0:.3f}s")
+
+        # vLLM batch generation
+        outputs = self.vllm_engine.generate(prompts, sampling_params, use_tqdm=len(inputs) > 1)
+
+        t2 = time.perf_counter()
+        logger.info(f"vLLM generation: {t2 - t1:.3f}s")
+
+        # Process results
+        results = []
+        for i, output in enumerate(outputs):
+            token_ids = list(output.outputs[0].token_ids)
+            text = self.tokenizer.decode(token_ids, skip_special_tokens=True)
+            # Clean vLLM artifacts: remove garbage prefix/tags
+            text = re.sub(r'<[^>]*>', '', text)
+            text = re.sub(r'\[[^\]]*\]', '', text)
+            text = re.sub(r'endofpatch|/sil|FFFF|</strong>', '', text)
+            # Strip non-CJK/non-alnum prefix garbage
+            text = re.sub(r'^[^\w一-鿿]+', '', text)
+            text_clean = re.sub(r"\s+", " ", text).strip()
+
+            key = (
+                os.path.splitext(os.path.basename(inputs[i]))[0]
+                if isinstance(inputs[i], str)
+                else f"sample_{i}"
+            )
+            result = {"key": key, "text": text_clean}
+
+            # Timestamps via CTC forced alignment
+            if self.ctc_decoder is not None and self.ctc_tokenizer is not None:
+                try:
+                    timestamps = self._compute_timestamps(
+                        encoder_outputs[i][0], encoder_outputs[i][1], text_clean
+                    )
+                    if timestamps:
+                        result["timestamps"] = timestamps
+                except Exception as e:
+                    logger.debug(f"Timestamp computation failed for {key}: {e}")
+
+            results.append(result)
+
+        return results
+
+    @torch.no_grad()
+    def _compute_timestamps(self, encoder_out, encoder_out_lens, text):
+        """CTC forced alignment for character-level timestamps."""
+        from funasr.models.fun_asr_nano.tools.utils import forced_align
+
+        decoder_out, decoder_out_lens = self.ctc_decoder(encoder_out, encoder_out_lens)
+        ctc_logits = self.ctc.log_softmax(decoder_out)
+        x = ctc_logits[0, : encoder_out_lens[0].item(), :]
+
+        target_ids = torch.tensor(self.ctc_tokenizer.encode(text), dtype=torch.int64)
+        if len(target_ids) == 0:
+            return []
+
+        timestamps = forced_align(x, target_ids, self.blank_id)
+        for ts in timestamps:
+            ts["token"] = self.ctc_tokenizer.decode([ts["token"]])
+            ts["start_time"] = ts["start_time"] * 6 * 10 / 1000
+            ts["end_time"] = ts["end_time"] * 6 * 10 / 1000
+        return timestamps
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        model: str = "FunAudioLLM/Fun-ASR-Nano-2512",
+        hub: str = "ms",
+        device: str = "cuda:0",
+        dtype: str = "bf16",
+        tensor_parallel_size: int = 1,
+        gpu_memory_utilization: float = 0.8,
+        max_model_len: int = 2048,
+        **kwargs,
+    ) -> "FunASRNanoVLLM":
+        """Load model from hub or local path.
+
+        Args:
+            model: Model name or local directory path.
+            hub: "ms" (ModelScope) or "hf" (HuggingFace).
+            device: Device for audio encoder/adaptor.
+            dtype: Compute dtype ("bf16", "fp16", "fp32").
+            tensor_parallel_size: GPUs for vLLM tensor parallel.
+            gpu_memory_utilization: GPU memory fraction for vLLM.
+            max_model_len: Maximum sequence length.
+
+        Returns:
+            Initialized FunASRNanoVLLM engine.
+        """
+        if os.path.isdir(model):
+            model_dir = model
+        else:
+            if hub in ("ms", "modelscope"):
+                from modelscope.hub.snapshot_download import snapshot_download
+
+                model_dir = snapshot_download(model, revision=kwargs.pop("revision", "master"))
+            elif hub in ("hf", "huggingface"):
+                from huggingface_hub import snapshot_download
+
+                model_dir = snapshot_download(model)
+            else:
+                raise ValueError(f"Unsupported hub: {hub}. Use 'ms' or 'hf'.")
+
+        logger.info(f"Model directory: {model_dir}")
+        return cls(
+            model_dir=model_dir,
+            device=device,
+            dtype=dtype,
+            tensor_parallel_size=tensor_parallel_size,
+            gpu_memory_utilization=gpu_memory_utilization,
+            max_model_len=max_model_len,
+            **kwargs,
+        )

funasr/models/fun_asr_nano/inference_vllm_pipeline.py

@@ -0,0 +1,372 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+"""
+Fun-ASR-Nano vLLM Pipeline: VAD + ASR(vLLM) + Speaker Diarization.
+
+Replicates AutoModel's inference_with_vad pipeline but uses vLLM for
+the LLM decoding step, enabling batch processing of all VAD segments
+in a single generate() call.
+
+Usage:
+    from funasr.models.fun_asr_nano.inference_vllm_pipeline import FunASRNanoVLLMPipeline
+
+    model = FunASRNanoVLLMPipeline(
+        model="FunAudioLLM/Fun-ASR-Nano-2512",
+        vad_model="fsmn-vad",
+        spk_model="cam++",
+        tensor_parallel_size=2,
+    )
+    results = model.generate("long_meeting.wav", language="中文")
+"""
+
+import logging
+import os
+import re
+import time
+from typing import List, Optional, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+logger = logging.getLogger(__name__)
+
+dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}
+
+
+def _clean_text(text: str) -> str:
+    """Remove tags, fillers, and garbage from output."""
+    text = re.sub(r"<[^>]*>|</[^>]*>", "", text)
+    text = re.sub(r"(>.{2,8}?)\1{3,}", "", text)
+    text = re.sub(r"\[breath\]|\[noise\]|/sil|endofbreak|FFFF", "", text)
+    text = re.sub(r"\s+", " ", text)
+    text = text.replace("�", "").lstrip(">")
+    return text.strip()
+
+
+class FunASRNanoVLLMPipeline:
+    """VAD + ASR(vLLM) + Speaker pipeline.
+
+    Pipeline:
+        1. VAD: segment long audio into speech regions (torch)
+        2. ASR: batch ALL segments through vLLM in single generate() call
+        3. Speaker: extract embeddings per segment, cluster (torch)
+        4. Combine: merge text + timestamps + speaker labels
+
+    Args:
+        model: Fun-ASR-Nano model name or path.
+        vad_model: VAD model name (e.g. "fsmn-vad"). None to disable.
+        vad_kwargs: VAD config (e.g. {"max_single_segment_time": 30000}).
+        spk_model: Speaker model name (e.g. "cam++"). None to disable.
+        hub: "ms" or "hf".
+        device: Device for audio encoder + VAD + speaker.
+        dtype: Compute dtype for ASR.
+        tensor_parallel_size: GPUs for vLLM.
+        gpu_memory_utilization: GPU memory fraction for vLLM.
+        max_model_len: Maximum sequence length for vLLM.
+    """
+
+    def __init__(
+        self,
+        model: str = "FunAudioLLM/Fun-ASR-Nano-2512",
+        vad_model: str = None,
+        vad_kwargs: dict = None,
+        spk_model: str = None,
+        spk_kwargs: dict = None,
+        hub: str = "ms",
+        device: str = "cuda:0",
+        dtype: str = "bf16",
+        tensor_parallel_size: int = 1,
+        gpu_memory_utilization: float = 0.8,
+        max_model_len: int = 4096,
+        enforce_eager: bool = False,
+        **kwargs,
+    ):
+        from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+
+        # ASR engine (vLLM)
+        self.asr_engine = FunASRNanoVLLM.from_pretrained(
+            model=model, hub=hub, device=device, dtype=dtype,
+            tensor_parallel_size=tensor_parallel_size,
+            gpu_memory_utilization=gpu_memory_utilization,
+            max_model_len=max_model_len, enforce_eager=enforce_eager,
+            **kwargs,
+        )
+
+        # VAD model (torch)
+        self.vad_model = None
+        if vad_model is not None:
+            from funasr import AutoModel
+            vad_kw = vad_kwargs or {}
+            self.vad_model = AutoModel(
+                model=vad_model, device=device, disable_update=True, **vad_kw
+            )
+
+        # Speaker model (torch)
+        self.spk_model = None
+        self.cb_model = None
+        if spk_model is not None:
+            from funasr import AutoModel
+            from funasr.models.campplus.cluster_backend import ClusterBackend
+            spk_kw = spk_kwargs or {}
+            self.spk_model = AutoModel(
+                model=spk_model, device=device, disable_update=True, **spk_kw
+            )
+            cb_kwargs = spk_kw.get("cb_kwargs", {})
+            self.cb_model = ClusterBackend(**cb_kwargs).to(device)
+
+        self.device = device
+        self.sample_rate = 16000
+
+    def generate(
+        self,
+        input: Union[str, List[str]],
+        hotwords: List[str] = None,
+        language: str = None,
+        itn: bool = True,
+        max_new_tokens: int = 512,
+        batch_size_s: int = 300,
+        return_spk_res: bool = True,
+        **kwargs,
+    ) -> List[dict]:
+        """Run the full pipeline: VAD → ASR(vLLM) → Speaker.
+
+        Args:
+            input: Audio file path(s).
+            hotwords: Hotwords for ASR.
+            language: Language hint.
+            itn: Inverse text normalization.
+            max_new_tokens: Max tokens per segment.
+            batch_size_s: Max batch duration in seconds (for memory control).
+            return_spk_res: Whether to return speaker info.
+
+        Returns:
+            List of dicts: [{"key", "text", "timestamp", "sentence_info"}]
+        """
+        if isinstance(input, str):
+            input = [input]
+
+        results_all = []
+        for audio_path in input:
+            result = self._process_one(
+                audio_path, hotwords=hotwords, language=language,
+                itn=itn, max_new_tokens=max_new_tokens,
+                batch_size_s=batch_size_s, return_spk_res=return_spk_res,
+                **kwargs,
+            )
+            results_all.append(result)
+        return results_all
+
+    def _process_one(self, audio_path, **kwargs):
+        """Process a single audio file through the full pipeline."""
+        from funasr.utils.load_utils import load_audio_text_image_video
+        from funasr.utils.vad_utils import slice_padding_audio_samples
+
+        key = os.path.splitext(os.path.basename(audio_path))[0]
+
+        # Load audio
+        audio_data = load_audio_text_image_video(audio_path, fs=self.sample_rate)
+        if isinstance(audio_data, torch.Tensor):
+            audio_np = audio_data.numpy()
+        else:
+            audio_np = np.array(audio_data)
+        speech_length = len(audio_np)
+
+        # Step 1: VAD
+        if self.vad_model is not None:
+            vad_res = self.vad_model.generate(input=audio_path, cache={}, is_final=True)
+            vad_segments = vad_res[0]["value"]  # [[start_ms, end_ms], ...]
+        else:
+            vad_segments = [[0, int(speech_length / self.sample_rate * 1000)]]
+
+        if not vad_segments:
+            return {"key": key, "text": "", "timestamp": []}
+
+        n_segments = len(vad_segments)
+        logger.info(f"VAD: {n_segments} segments for {key}")
+
+        # Step 2: Slice audio by VAD segments and encode
+        segment_audios = []
+        for seg in vad_segments:
+            start_sample = int(seg[0] * self.sample_rate / 1000)
+            end_sample = int(seg[1] * self.sample_rate / 1000)
+            end_sample = min(end_sample, speech_length)
+            segment_audios.append(audio_np[start_sample:end_sample])
+
+        # Step 3: Batch ASR via vLLM
+        # Encode all segments and build prompts
+        from vllm import SamplingParams
+        try:
+            from vllm.inputs import EmbedsPrompt
+        except ImportError:
+            from vllm.inputs.data import EmbedsPrompt
+
+        from funasr.models.fun_asr_nano.vllm_utils import resolve_repetition_penalty
+
+        prompts = []
+        for seg_audio in segment_audios:
+            seg_tensor = torch.from_numpy(seg_audio).float()
+            adaptor_out, adaptor_out_lens, _, _ = self.asr_engine._encode_audio(seg_tensor)
+            input_embeds = self.asr_engine._build_input_embeds(
+                adaptor_out, adaptor_out_lens,
+                hotwords=kwargs.get("hotwords"),
+                language=kwargs.get("language"),
+                itn=kwargs.get("itn", True),
+            )
+            prompts.append(EmbedsPrompt(prompt_embeds=input_embeds.float()))
+
+        params = SamplingParams(
+            max_tokens=kwargs.get("max_new_tokens", 512),
+            temperature=0.0,
+            # Prompt-embeds mode has no token IDs to penalize; see #2948.
+            repetition_penalty=resolve_repetition_penalty(
+                kwargs.get("repetition_penalty", 1.0)
+            ),
+            skip_special_tokens=True,
+        )
+
+        # Single batch generate for ALL segments
+        t0 = time.perf_counter()
+        outputs = self.asr_engine.vllm_engine.generate(prompts, params, use_tqdm=False)
+        t1 = time.perf_counter()
+        logger.info(f"vLLM batch ASR: {n_segments} segments in {t1-t0:.3f}s")
+
+        # Decode results
+        asr_results = []
+        for output in outputs:
+            text = output.outputs[0].text
+            if not text and output.outputs[0].token_ids:
+                text = self.asr_engine.tokenizer.decode(
+                    list(output.outputs[0].token_ids), skip_special_tokens=True
+                )
+            text = _clean_text(text)
+            asr_results.append(text)
+
+        # Step 4: Speaker embeddings (if spk_model configured)
+        spk_embeddings = None
+        if self.spk_model is not None and kwargs.get("return_spk_res", True):
+            from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk
+
+            all_segments = []
+            all_spk_embs = []
+            for i, seg_audio in enumerate(segment_audios):
+                vad_seg = [
+                    [vad_segments[i][0] / 1000.0, vad_segments[i][1] / 1000.0, seg_audio]
+                ]
+                chunks = sv_chunk(vad_seg)
+                all_segments.extend(chunks)
+                speech_chunks = [c[2] for c in chunks]
+                spk_res = self.spk_model.generate(input=speech_chunks, cache={}, is_final=True)
+                embs = torch.cat([r["spk_embedding"] for r in spk_res], dim=0)
+                all_spk_embs.append(embs)
+
+            if all_spk_embs:
+                spk_embeddings = torch.cat(all_spk_embs, dim=0)
+
+        # Step 5: Combine results
+        # Merge text with timestamps
+        full_text = ""
+        all_timestamps = []
+        for i, (seg, text) in enumerate(zip(vad_segments, asr_results)):
+            if text:
+                if full_text:
+                    full_text += " "
+                full_text += text
+                # Simple word-level timestamp from VAD boundaries
+                all_timestamps.append([int(seg[0]), int(seg[1])])
+
+        result = {"key": key, "text": full_text}
+
+        # Add timestamps if available from CTC
+        if self.asr_engine.ctc_decoder is not None:
+            try:
+                detailed_timestamps = self._compute_all_timestamps(
+                    segment_audios, vad_segments, asr_results
+                )
+                if detailed_timestamps:
+                    result["timestamp"] = detailed_timestamps
+            except Exception as e:
+                logger.debug(f"Timestamp computation failed: {e}")
+                result["timestamp"] = all_timestamps
+        else:
+            result["timestamp"] = all_timestamps
+
+        # Add speaker info
+        if spk_embeddings is not None and self.cb_model is not None:
+            from funasr.models.campplus.utils import postprocess, distribute_spk
+
+            all_segments_sorted = sorted(all_segments, key=lambda x: x[0])
+            labels = self.cb_model(
+                spk_embeddings.cpu(),
+                oracle_num=kwargs.get("preset_spk_num", None),
+            )
+            sv_output = postprocess(all_segments_sorted, None, labels, spk_embeddings.cpu())
+
+            # Build sentence_info
+            sentence_list = []
+            for i, (seg, text) in enumerate(zip(vad_segments, asr_results)):
+                if text:
+                    sentence_list.append({
+                        "start": seg[0],
+                        "end": seg[1],
+                        "text": text,
+                        "timestamp": [[int(seg[0]), int(seg[1])]],
+                    })
+            distribute_spk(sentence_list, sv_output)
+            result["sentence_info"] = sentence_list
+
+        return result
+
+    def _compute_all_timestamps(self, segment_audios, vad_segments, asr_results):
+        """Compute CTC timestamps for all segments with VAD offsets."""
+        from funasr.models.fun_asr_nano.tools.utils import forced_align
+
+        all_timestamps = []
+        for seg_audio, vad_seg, text in zip(segment_audios, vad_segments, asr_results):
+            if not text:
+                continue
+            try:
+                seg_tensor = torch.from_numpy(seg_audio).float()
+                from funasr.utils.load_utils import extract_fbank
+                speech, speech_lengths = extract_fbank(
+                    seg_tensor, data_type="sound",
+                    frontend=self.asr_engine.frontend, is_final=True
+                )
+                speech = speech.to(self.device, dtype=torch.float32)
+                speech_lengths = speech_lengths.to(self.device)
+
+                with torch.no_grad():
+                    enc_out, enc_lens = self.asr_engine.audio_encoder(speech, speech_lengths)
+                    dec_out, dec_lens = self.asr_engine.ctc_decoder(enc_out, enc_lens)
+                    ctc_logits = self.asr_engine.ctc.log_softmax(dec_out)
+
+                x = ctc_logits[0, :enc_lens[0].item(), :]
+                target_ids = torch.tensor(
+                    self.asr_engine.ctc_tokenizer.encode(text), dtype=torch.int64
+                )
+                if len(target_ids) == 0:
+                    continue
+
+                timestamps = forced_align(x, target_ids, self.asr_engine.blank_id)
+                vad_offset_ms = int(vad_seg[0])
+                for ts in timestamps:
+                    ts["token"] = self.asr_engine.ctc_tokenizer.decode([ts["token"]])
+                    ts["start_time"] = ts["start_time"] * 6 * 10 / 1000 + vad_offset_ms / 1000
+                    ts["end_time"] = ts["end_time"] * 6 * 10 / 1000 + vad_offset_ms / 1000
+                all_timestamps.extend(timestamps)
+            except Exception as e:
+                logger.debug(f"Timestamp failed for segment: {e}")
+                all_timestamps.append({
+                    "start_time": vad_seg[0] / 1000,
+                    "end_time": vad_seg[1] / 1000,
+                    "token": text,
+                })
+        return all_timestamps
+
+    @classmethod
+    def from_pretrained(cls, model="FunAudioLLM/Fun-ASR-Nano-2512", **kwargs):
+        """Convenience constructor."""
+        return cls(model=model, **kwargs)

funasr/models/fun_asr_nano/inference_vllm_streaming.py

@@ -0,0 +1,350 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+"""
+Fun-ASR-Nano Streaming vLLM Inference Engine.
+
+Design:
+    - Audio split into 720ms chunks (cumulative re-encoding)
+    - ALL chunks batched into single vLLM generate call for correctness
+    - Fixed/Unfixed: last 8 chars are unfixed (may change on next chunk)
+    - Output stabilizes as more audio accumulates (~3s+)
+
+Note: vLLM processes all chunks in one batch for throughput.
+For real-time streaming, use the torch-based inference in demo2.py.
+"""
+
+import logging
+import os
+import re
+from typing import Generator, List, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+logger = logging.getLogger(__name__)
+dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}
+
+_CJK_RE = re.compile(r"[一-鿿]")
+
+
+def _clean_text(text: str) -> str:
+    """Remove tags, repetitive garbage, filler tokens, and invalid chars."""
+    text = re.sub(r'<[^>]*>|</[^>]*>', '', text)
+    text = re.sub(r'(>.{2,8}?)
{3,}', '', text)
+    text = re.sub(r'\[breath\]|\[noise\]|/sil|endofbreak|FFFF', '', text)
+    text = re.sub(r'\s+', ' ', text)
+    text = text.replace('�', '').lstrip('>')
+    return text.strip()
+
+
+def _is_meaningful(text: str) -> bool:
+    """Check if text has real ASR content."""
+    return len(_CJK_RE.findall(text)) >= 2
+
+
+class FunASRNanoStreamingVLLM:
+    """Streaming ASR with vLLM backend (batch-all-chunks approach).
+
+    Processes audio in 720ms chunks. All chunks are encoded and batched
+    into a single vLLM generate() call for correct and efficient inference.
+    Results are returned per-chunk with fixed/unfixed regions.
+
+    Args:
+        model_dir: Path to Fun-ASR-Nano model directory.
+        device: Device for audio encoder/adaptor.
+        dtype: Compute dtype ("bf16", "fp16", "fp32").
+        tensor_parallel_size: GPUs for vLLM tensor parallelism.
+        gpu_memory_utilization: GPU memory fraction for KV cache.
+        max_model_len: Maximum sequence length.
+        chunk_ms: Chunk duration in ms (default 720).
+        rollback_chars: Characters to rollback per chunk (default 8).
+    """
+
+    def __init__(self, model_dir, device="cuda:0", dtype="bf16",
+                 tensor_parallel_size=1, gpu_memory_utilization=0.8,
+                 max_model_len=2048, enforce_eager=False,
+                 chunk_ms=720, rollback_chars=8, **kwargs):
+        from vllm import LLM
+        from funasr.models.fun_asr_nano.inference_vllm import prepare_vllm_model_dir
+
+        self.device = device
+        self.dtype = dtype
+        self.torch_dtype = dtype_map.get(dtype, torch.bfloat16)
+        self.model_dir = model_dir
+        self.chunk_ms = chunk_ms
+        self.rollback_chars = rollback_chars
+
+        vllm_model_dir = prepare_vllm_model_dir(model_dir)
+        self._load_audio_components(model_dir)
+
+        vllm_kwargs = kwargs.get("vllm_kwargs", {})
+        self.vllm_engine = LLM(
+            enable_prompt_embeds=True, model=vllm_model_dir,
+            tensor_parallel_size=tensor_parallel_size,
+            gpu_memory_utilization=gpu_memory_utilization,
+            max_model_len=max_model_len, enforce_eager=enforce_eager,
+            dtype={"bf16": "bfloat16", "fp16": "float16", "fp32": "auto"}.get(dtype, dtype),
+            trust_remote_code=True, **vllm_kwargs,
+        )
+        self.tokenizer = self.vllm_engine.get_tokenizer()
+        self._load_embedding_layer(model_dir)
+        self.sample_rate = self.frontend.fs
+        self.chunk_samples = int(self.sample_rate * self.chunk_ms / 1000)
+
+    def _load_audio_components(self, model_dir):
+        from omegaconf import OmegaConf
+        from funasr.register import tables
+        config = OmegaConf.load(os.path.join(model_dir, "config.yaml"))
+        self._config = OmegaConf.to_container(config, resolve=True)
+
+        frontend_class = tables.frontend_classes.get(config["frontend"])
+        frontend_conf = OmegaConf.to_container(config.get("frontend_conf", {}), resolve=True)
+        self.frontend = frontend_class(**frontend_conf)
+        self.frontend.eval()
+
+        encoder_conf = OmegaConf.to_container(config.get("audio_encoder_conf", {}), resolve=True)
+        if encoder_conf.get("hub") == "ms":
+            from funasr import AutoModel as FAM
+            enc_m = FAM(model=config["audio_encoder"], model_revision="master", disable_update=True)
+            self.audio_encoder_output_size = getattr(enc_m.model, "encoder_output_size", -1)
+            self.audio_encoder = enc_m.model.model.encoder if hasattr(enc_m.model, "model") else enc_m.model.encoder
+        else:
+            encoder_class = tables.encoder_classes.get(config["audio_encoder"])
+            self.audio_encoder = encoder_class(input_size=self.frontend.output_size(), **encoder_conf)
+            self.audio_encoder_output_size = self.audio_encoder.output_size()
+        self.audio_encoder.eval()
+        for p in self.audio_encoder.parameters(): p.requires_grad = False
+
+        adaptor_conf = OmegaConf.to_container(config.get("audio_adaptor_conf", {}), resolve=True)
+        adaptor_class = tables.adaptor_classes.get(config["audio_adaptor"])
+        if self.audio_encoder_output_size > 0:
+            adaptor_conf["encoder_dim"] = self.audio_encoder_output_size
+        self.audio_adaptor = adaptor_class(**adaptor_conf)
+        self.audio_adaptor.eval()
+        for p in self.audio_adaptor.parameters(): p.requires_grad = False
+
+        model_pt = os.path.join(model_dir, "model.pt")
+        if os.path.exists(model_pt):
+            ckpt = torch.load(model_pt, map_location="cpu")
+            sd = ckpt.get("state_dict", ckpt)
+            enc_s = {k[len("audio_encoder."):]: v for k, v in sd.items() if k.startswith("audio_encoder.")}
+            if enc_s: self.audio_encoder.load_state_dict(enc_s, strict=False)
+            adp_s = {k[len("audio_adaptor."):]: v for k, v in sd.items() if k.startswith("audio_adaptor.")}
+            if adp_s: self.audio_adaptor.load_state_dict(adp_s, strict=False)
+
+        self.audio_encoder = self.audio_encoder.to(self.device, dtype=torch.float32)
+        self.audio_adaptor = self.audio_adaptor.to(self.device, dtype=self.torch_dtype)
+
+    def _load_embedding_layer(self, model_dir):
+        ckpt = torch.load(os.path.join(model_dir, "model.pt"), map_location="cpu")
+        sd = ckpt.get("state_dict", ckpt)
+        for key in sd:
+            if "embed_tokens.weight" in key and key.startswith("llm."):
+                self.embed_tokens = nn.Embedding.from_pretrained(sd[key], freeze=True)
+                self.embed_tokens = self.embed_tokens.to(self.device, dtype=self.torch_dtype)
+                return
+        raise RuntimeError("Could not find LLM embedding weights")
+
+    @torch.no_grad()
+    def _encode_audio(self, audio_samples):
+        from funasr.utils.load_utils import extract_fbank
+        speech, speech_lengths = extract_fbank(
+            audio_samples, data_type="sound", frontend=self.frontend, is_final=True)
+        speech = speech.to(self.device, dtype=torch.float32)
+        speech_lengths = speech_lengths.to(self.device)
+        enc_out, enc_lens = self.audio_encoder(speech, speech_lengths)
+        adp_out, adp_lens = self.audio_adaptor(enc_out.to(dtype=self.torch_dtype), enc_lens)
+        return adp_out, adp_lens
+
+    def _build_prompt_text(self, hotwords=None, language=None, itn=True):
+        hotwords = hotwords or []
+        if hotwords:
+            prompt = "请结合上下文信息，更加准确地完成语音转写任务。如果没有相关信息，我们会留空。\n\n\n**上下文信息：**\n\n\n"
+            prompt += f"热词列表：[{', '.join(hotwords)}]\n"
+        else:
+            prompt = ""
+        prompt += f"语音转写成{language}" if language else "语音转写"
+        if not itn: prompt += "，不进行文本规整"
+        return prompt + "："
+
+    @torch.no_grad()
+    def _build_embeds(self, audio_embeds, audio_embed_lens, prev_text="", hotwords=None, language=None, itn=True):
+        """Build input embeddings. prev_text is appended as assistant prefix for continuation."""
+        prompt = self._build_prompt_text(hotwords, language, itn)
+        prefix_text = f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{prompt}<|startofspeech|>"
+        suffix_text = "<|endofspeech|><|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
+        if prev_text:
+            suffix_text += prev_text
+
+        prefix_ids = self.tokenizer.encode(prefix_text, add_special_tokens=False)
+        suffix_ids = self.tokenizer.encode(suffix_text, add_special_tokens=False)
+        prefix_emb = self.embed_tokens(torch.tensor(prefix_ids, dtype=torch.long, device=self.device))
+        suffix_emb = self.embed_tokens(torch.tensor(suffix_ids, dtype=torch.long, device=self.device))
+        audio_emb = audio_embeds[0, :audio_embed_lens[0].item(), :]
+        return torch.cat([prefix_emb, audio_emb, suffix_emb], dim=0)
+
+    def streaming_generate(self, audio_input, chunk_ms=None, rollback_chars=None,
+                           hotwords=None, language=None, itn=True,
+                           max_new_tokens=200, temperature=0.0, **kwargs):
+        """Streaming ASR: process all chunks and yield results per chunk.
+
+        All chunks are batched into a single vLLM generate() call for
+        correct results. Yields incrementally improving transcriptions.
+
+        Args:
+            audio_input: File path, numpy array, or tensor (16kHz).
+            chunk_ms: Chunk size in ms (default 720).
+            rollback_chars: Chars to rollback (default 8).
+            hotwords: Hotword list.
+            language: Language hint (e.g. "中文").
+            itn: Inverse text normalization.
+            max_new_tokens: Max tokens per chunk generation.
+            temperature: Sampling temperature (0 = greedy).
+
+        Yields:
+            {"text": full_text, "fixed_text": confirmed_text,
+             "is_final": bool, "chunk_idx": int, "audio_duration_ms": float}
+        """
+        from vllm import SamplingParams
+        try:
+            from vllm.inputs import EmbedsPrompt
+        except ImportError:
+            from vllm.inputs.data import EmbedsPrompt
+        from funasr.utils.load_utils import load_audio_text_image_video
+
+        chunk_ms = chunk_ms or self.chunk_ms
+        rollback_chars = rollback_chars or self.rollback_chars
+
+        if isinstance(audio_input, str):
+            audio_data = load_audio_text_image_video(audio_input, fs=self.sample_rate)
+        elif isinstance(audio_input, np.ndarray):
+            audio_data = torch.from_numpy(audio_input).float()
+        elif isinstance(audio_input, torch.Tensor):
+            audio_data = audio_input.float()
+        else:
+            raise ValueError(f"Unsupported audio type: {type(audio_input)}")
+        if audio_data.dim() > 1:
+            audio_data = audio_data.squeeze()
+
+        total_samples = audio_data.shape[0]
+        chunk_samples = int(self.sample_rate * chunk_ms / 1000)
+        num_chunks = (total_samples + chunk_samples - 1) // chunk_samples
+
+        from funasr.models.fun_asr_nano.vllm_utils import resolve_repetition_penalty
+
+        # Prompt-embeds mode has no token IDs to penalize; see #2948.
+        params = SamplingParams(
+            max_tokens=max_new_tokens, temperature=temperature,
+            repetition_penalty=resolve_repetition_penalty(
+                kwargs.get("repetition_penalty", 1.0)
+            ),
+            skip_special_tokens=True)
+
+        # Two-stage approach for long audio:
+        # Stage 1: batch first N chunks fresh (no prev_text) to find stable output
+        # Stage 2: batch remaining chunks WITH prev_text from stable output
+        stage1_count = min(10, num_chunks)  # ~7.2s should be enough to stabilize
+
+        # Stage 1: encode and batch first chunks
+        prompts_s1 = []
+        chunk_infos_s1 = []
+        for i in range(stage1_count):
+            end_sample = min((i + 1) * chunk_samples, total_samples)
+            adaptor_out, adaptor_out_lens = self._encode_audio(audio_data[:end_sample])
+            embeds = self._build_embeds(adaptor_out, adaptor_out_lens, prev_text="",
+                                        hotwords=hotwords, language=language, itn=itn)
+            prompts_s1.append(EmbedsPrompt(prompt_embeds=embeds.float()))
+            chunk_infos_s1.append({
+                "chunk_idx": i + 1,
+                "is_final": end_sample >= total_samples,
+                "audio_duration_ms": end_sample * 1000 / self.sample_rate,
+            })
+
+        outputs_s1 = self.vllm_engine.generate(prompts_s1, params, use_tqdm=False)
+
+        # Find best stable output from stage 1
+        best_text = ""
+        results_s1 = []
+        for output in outputs_s1:
+            text = output.outputs[0].text
+            if not text and output.outputs[0].token_ids:
+                text = self.tokenizer.decode(list(output.outputs[0].token_ids), skip_special_tokens=True)
+            text = _clean_text(text)
+            results_s1.append(text)
+            if _is_meaningful(text) and len(text) > len(best_text):
+                best_text = text
+
+        # Yield stage 1 results
+        for i, (text, info) in enumerate(zip(results_s1, chunk_infos_s1)):
+            if info["is_final"]:
+                fixed_text = text
+            elif _is_meaningful(text) and len(text) > rollback_chars:
+                fixed_text = text[:-rollback_chars]
+            else:
+                fixed_text = ""
+            yield {"text": text, "fixed_text": fixed_text, **info}
+
+        # Stage 2: if more chunks remain, use prev_text from stable output
+        if stage1_count < num_chunks:
+            prev_text = best_text[:-rollback_chars] if len(best_text) > rollback_chars else best_text
+
+            prompts_s2 = []
+            chunk_infos_s2 = []
+            for i in range(stage1_count, num_chunks):
+                end_sample = min((i + 1) * chunk_samples, total_samples)
+                adaptor_out, adaptor_out_lens = self._encode_audio(audio_data[:end_sample])
+                embeds = self._build_embeds(adaptor_out, adaptor_out_lens, prev_text=prev_text,
+                                            hotwords=hotwords, language=language, itn=itn)
+                prompts_s2.append(EmbedsPrompt(prompt_embeds=embeds.float()))
+                chunk_infos_s2.append({
+                    "chunk_idx": i + 1,
+                    "is_final": end_sample >= total_samples,
+                    "audio_duration_ms": end_sample * 1000 / self.sample_rate,
+                })
+
+            outputs_s2 = self.vllm_engine.generate(prompts_s2, params, use_tqdm=False)
+
+            for output, info in zip(outputs_s2, chunk_infos_s2):
+                text = output.outputs[0].text
+                if not text and output.outputs[0].token_ids:
+                    text = self.tokenizer.decode(list(output.outputs[0].token_ids), skip_special_tokens=True)
+                text = _clean_text(text)
+                full_text = prev_text + text
+
+                if info["is_final"]:
+                    fixed_text = full_text
+                elif _is_meaningful(full_text) and len(full_text) > rollback_chars:
+                    fixed_text = full_text[:-rollback_chars]
+                else:
+                    fixed_text = prev_text
+
+                yield {"text": full_text, "fixed_text": fixed_text, **info}
+
+    def generate(self, audio_input, **kwargs):
+        """Run streaming and return all chunk results."""
+        return list(self.streaming_generate(audio_input, **kwargs))
+
+    @classmethod
+    def from_pretrained(cls, model="FunAudioLLM/Fun-ASR-Nano-2512", hub="ms",
+                        device="cuda:0", dtype="bf16", tensor_parallel_size=1,
+                        gpu_memory_utilization=0.8, max_model_len=2048,
+                        chunk_ms=720, rollback_chars=8, **kwargs):
+        """Load from hub or local path."""
+        if os.path.isdir(model):
+            model_dir = model
+        else:
+            if hub in ("ms", "modelscope"):
+                from modelscope.hub.snapshot_download import snapshot_download
+                model_dir = snapshot_download(model, revision=kwargs.pop("revision", "master"))
+            else:
+                from huggingface_hub import snapshot_download
+                model_dir = snapshot_download(model)
+        return cls(model_dir=model_dir, device=device, dtype=dtype,
+                   tensor_parallel_size=tensor_parallel_size,
+                   gpu_memory_utilization=gpu_memory_utilization,
+                   max_model_len=max_model_len, chunk_ms=chunk_ms,
+                   rollback_chars=rollback_chars, **kwargs)

funasr/models/fun_asr_nano/tools/format5res.py

@@ -0,0 +1,331 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/python
+# Author: Mengze Chen
+
+import re
+import sys
+
+
+def scoreformat(name, line, flag=1):
+    """Scoreformat.
+    
+        Args:
+            name: TODO.
+            line: TODO.
+            flag: TODO.
+        """
+    newline = ""
+    for i in range(0, len(line)):
+        curr = line[i]
+        currEn = False
+        if curr == "":
+            continue
+        if (
+            (curr >= "\u0041" and curr <= "\u005a")  # eng
+            or (curr >= "\u0061" and curr <= "\u007a")  # eng
+            or (curr >= "\u0000" and curr <= "\u007f")  # de fr es it
+            or (curr >= "\u0400" and curr <= "\u04ff")  # ru
+            or (curr >= "\u0100" and curr <= "\u017f")  # latin1
+            or (curr >= "\u0080" and curr <= "\u00ff")  # latin2
+            or curr == "'"
+        ) and (curr < "\u0030" or curr > "\u0039"):
+            currEn = True
+        if i == 0:
+            newline = newline + curr
+        else:
+            if lastEn == True and currEn == True:
+                newline = newline + curr
+            else:
+                newline = newline + " " + curr
+        if flag == -1:
+            lastEn = False
+        else:
+            lastEn = currEn
+    ret = re.sub("[ ]{1,}", " ", newline)
+    ret = ret
+    if name == "":
+        ret = ret
+    else:
+        if flag <= 0:
+            ret = ret + " " + "(" + name + ")"
+        else:
+            ret = name + "\t" + ret
+    return ret
+
+
+def recoformat(line):
+    """Recoformat.
+    
+        Args:
+            line: TODO.
+        """
+    newline = ""
+    en_flag = 0  # 0: no-english   1 : english   2: former
+    for i in range(0, len(line)):
+        word = line[i]
+        if ord(word) == 32:
+            if en_flag == 0:
+                continue
+            else:
+                en_flag = 0
+                newline += " "
+        if (word >= "\u4e00" and word <= "\u9fa5") or (word >= "\u0030" and word <= "\u0039"):
+            if en_flag == 1:
+                newline += " " + word
+            else:
+                newline += word
+            en_flag = 0
+        elif (
+            (word >= "\u0041" and word <= "\u005a")  # eng
+            or (word >= "\u0061" and word <= "\u007a")  # eng
+            or (word >= "\u0000" and word <= "\u007f")  # de fr es it
+            or (word >= "\u0400" and word <= "\u04ff")  # ru
+            or (word >= "\u0100" and word <= "\u017f")  # latin1
+            or (word >= "\u0080" and word <= "\u00ff")  # latin2
+            or word == "'"
+        ):
+            if en_flag == 0:
+                newline += " " + ("" if (word == "'") else word)
+            else:
+                newline += word
+            en_flag = 1
+        else:
+            newline += " " + word
+    newline = newline
+    newline = re.sub("[ ]{1,}", " ", newline)
+    newline = newline
+    return newline
+
+
+def numbersingle(line):
+    """Numbersingle.
+    
+        Args:
+            line: TODO.
+        """
+    chnu = ["零", "一", "二", "两", "三", "四", "五", "六", "七", "八", "九", "点"]
+    newline = ""
+    for id in range(len(line)):
+        if re.findall(r"\.", line[id]):
+            if re.findall(r"\.\s*$", line[id]):
+                newline += "."
+            else:
+                newline += chnu[10]
+        elif re.search(r"0", line[id]):
+            if id > 0 and id < len(line) - 1:
+                if (
+                    re.search(r"\d", line[id - 1])
+                    and (not re.search(r"\d", line[id + 1]))
+                    and (not re.search(r"0", line[id - 1]))
+                ):
+                    if id > 2 and len(line) > 2 and (not re.search(r"\d", line[id - 1])):
+                        newline = newline[:-1]
+                        newline += chnu[int(line[id - 1])] + "十"
+                    else:
+                        newline += chnu[int(line[id])]
+                else:
+                    newline += chnu[int(line[id])]
+            else:
+                newline += chnu[int(line[id])]
+        elif re.search(r"\d", line[id]):
+            newline += chnu[int(line[id])]
+        else:
+            newline += line[id]
+    return newline
+
+
+def ch_number2digit(line):
+    """Ch number2digit.
+    
+        Args:
+            line: TODO.
+        """
+    number_flag = 0
+    zero_flag = 0
+    bits = {
+        "零": "1",
+        "十": "2",
+        "百": "3",
+        "千": "4",
+        "万": "5",
+        "十万": "6",
+        "百万": "7",
+        "千万": "8",
+    }
+    chsh = {
+        "一": "1",
+        "二": "2",
+        "三": "3",
+        "四": "4",
+        "五": "5",
+        "六": "6",
+        "七": "7",
+        "八": "8",
+        "九": "9",
+        "两": "2",
+        "幺": "1",
+    }
+    unit = {"里": "1", "克": "1", "米": "1"}
+    newline = ""
+    digit = []
+    bit = []
+    onebit = ""
+    for i in range(len(line)):
+        if ord(line[i]) == 32:
+            newline += " "
+            continue
+        if line[i] in chsh:
+            number_flag = 1
+            if line[i] == "两":
+                if (i == len(line) - 1) or ((line[i + 1] not in chsh.keys()) and (line[i + 1] not in bits.keys())):
+                    number_flag = -1
+            if number_flag == 1:
+                digit.append(chsh[line[i]])
+
+        elif "十" == line[i] and number_flag == 0:
+            number_flag = 2
+            digit.append("1")
+            bit.append(line[i])
+        elif "十" == line[i] and number_flag == 3:
+            digit.append("1")
+            bit.append(line[i])
+        elif ("零" == line[i]) and (number_flag == 0 or number_flag == 1):
+            digit.append("0")
+        elif ("零" == line[i]) and number_flag == 3:
+            zero_flag = 1
+        elif number_flag == 1 and line[i] in bits:
+            number_flag = 3
+            if line[i] == "千":
+                if i < len(line) - 1:
+                    if line[i + 1] in unit:
+                        number_flag = -1
+            if number_flag == 3:
+                onebit = line[i]
+                bit.append(onebit)
+        elif number_flag == 3 and line[i] in bits:
+            onebit = bit[-1] + line[i]
+            if onebit in bits:
+                bit[-1] = onebit
+            else:
+                number_flag = -2
+        else:
+            number_flag = -1
+        if len(digit) > 0 and number_flag == -1:
+            number_flag = -2
+        if i == (len(line) - 1) and number_flag >= 0:
+            number_flag = -1
+        if number_flag < 0:
+            newdigit = ""
+            if len(digit) > 0:  # and (len(digit) == len(bit))):
+                if len(bit) == 1 and zero_flag == 0 and bit[0] == "百" and len(bit) != len(digit):
+                    bit.append("十")
+                if len(digit) == (len(bit) + 1):
+                    bit.append("零")
+                if len(digit) == len(bit):
+                    for m in range(len(digit))[-1::-1]:
+                        if int(bits[bit[m]]) == int(len(newdigit) + 1):
+                            newdigit += digit[m]
+                        else:
+                            nu = int(bits[bit[m]]) - len(newdigit) - 1
+                            for n in range(nu):
+                                newdigit += "0"
+                            newdigit += digit[m]
+                    for z in range(len(newdigit))[-1::-1]:
+                        newline += newdigit[z]
+                else:
+                    newline += "".join(digit)
+                bit = []
+                digit = []
+                zero_flag = 0
+            else:
+                newline += line[i]
+            if number_flag == -2:
+                newline += line[i]
+            number_flag = 0
+    return newline
+
+
+def special(line):
+    """Special.
+    
+        Args:
+            line: TODO.
+        """
+    newline = ""
+    for e in range(len(line)):
+        if ord(line[e]) == 247:
+            newline += "除以"
+        elif ord(line[e]) == 215:
+            newline += "乘以"
+        elif ord(line[e]) == 61:
+            newline += "等于"
+        elif ord(line[e]) == 43:
+            newline += "加"
+        elif ord(line[e]) == 45:
+            newline += "负"
+        elif ord(line[e]) == 8451:
+            newline += "摄氏度"
+        elif ord(line[e]) == 13217:
+            newline += "平方米"
+        elif ord(line[e]) == 8240 or ord(line[e]) == 65130:
+            newline += "%"
+        elif ord(line[e]) == 46:
+            newline += "点"
+        elif ord(line[e]) == 176:
+            newline += "度"
+            angel = 1
+        elif ord(line[e]) == 8242 and angel == 1:
+            newline += "分"
+        else:
+            newline += line[e]
+    return newline
+
+
+def all_convert(content):
+    """All convert.
+    
+        Args:
+            content: TODO.
+        """
+    content = recoformat(content)
+    content = numbersingle(content)
+    content = ch_number2digit(content)
+    content = special(content)
+    content = scoreformat("", content)
+    return content
+
+
+if __name__ == "__main__":
+    if len(sys.argv[1:]) < 1:
+        sys.stderr.write("Usage:\n .py  reco.result\n")
+        sys.stderr.write(" reco.result:   id<tab>recoresult\n")
+        sys.exit(1)
+    f = open(sys.argv[1])
+    flag = 0
+    if len(sys.argv[1:]) > 1:
+        flag = int(sys.argv[2])
+    for line in f.readlines():
+        if not line:
+            continue
+        line = line.rstrip()
+        tmp = line.split("\t")
+        if len(tmp) < 2:
+            tmp = line.split(",")
+            if len(tmp) < 2:
+                tmp = line.split(" ", 1)
+                if len(tmp) < 2:
+                    name = tmp[0]
+                    content = ""
+                    print(content)
+                    continue
+        name = tmp[0]
+        content = tmp[1]
+        name = re.sub("\.pcm", "", name)
+        name = re.sub("\.wav", "", name)
+        content = recoformat(content)
+        content = numbersingle(content)
+        content = ch_number2digit(content)
+        content = special(content)
+        content = scoreformat(name, content, flag)
+        print(content)
+    f.close()

funasr/models/fun_asr_nano/tools/scp2jsonl.py

@@ -0,0 +1,157 @@
+import hydra
+import json
+import os
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from io import BytesIO
+from typing import Dict, Optional, Tuple
+from urllib.request import urlopen
+
+import soundfile as sf
+from modelscope import AutoTokenizer
+from tqdm import tqdm
+from omegaconf import DictConfig, OmegaConf, ListConfig
+
+
+class LineProcessor:
+    def __init__(self, tokenizer):
+        """Initialize LineProcessor.
+        
+            Args:
+                tokenizer: Tokenizer instance for text encoding/decoding.
+            """
+        self.tokenizer = tokenizer
+        self.lock = threading.Lock()
+
+    def process_line(self, line_pair: Tuple[str, str]) -> Optional[Dict]:
+        """Process line.
+        
+            Args:
+                line_pair: TODO.
+            """
+        line1, line2 = line_pair
+
+        line1, line2 = line1.strip(), line2.strip()
+        if not line1 or not line2:
+            return None
+
+        parts1, parts2 = line1.split(maxsplit=1), line2.split(maxsplit=1)
+        if len(parts1) != 2 or len(parts2) != 2:
+            return None
+
+        utt1, utt2 = parts1[0], parts2[0]
+        wav_path, text = parts1[1], parts2[1]
+
+        if utt1 != utt2:
+            return {"error": f"UTT mismatch: {utt1} vs {utt2}"}
+
+        try:
+            if wav_path.startswith("http"):
+                response = urlopen(wav_path)
+                if response.status != 200:
+                    return {"error": f"WAV not found: {wav_path}"}
+                audio_file = BytesIO(response.read())
+                duration = sf.info(audio_file).duration
+            else:
+                if not os.path.exists(wav_path):
+                    return {"error": f"WAV not found: {wav_path}"}
+                duration = sf.info(wav_path).duration
+
+            data = {
+                "messages": [
+                    {"role": "system", "content": "You are a helpful assistant."},
+                    {
+                        "role": "user",
+                        "content": f"语音转写：<|startofspeech|>!{wav_path}<|endofspeech|>",
+                    },
+                    {"role": "assistant", "content": text},
+                ],
+                "speech_length": int((duration * 1000 - 25) // 10 + 1),
+                "text_length": len(self.tokenizer.tokenize(text)),
+            }
+            return {"success": data, "utt": utt1}
+
+        except Exception as e:
+            return {"error": f"Error processing {wav_path}: {str(e)}"}
+
+
+@hydra.main(config_name=None, version_base=None)
+def main_hydra(cfg: DictConfig):
+    """Main hydra.
+    
+        Args:
+            cfg: Configuration overrides.
+        """
+    def to_plain_list(cfg_item):
+        """To plain list.
+        
+            Args:
+                cfg_item: TODO.
+            """
+        if isinstance(cfg_item, ListConfig):
+            return OmegaConf.to_container(cfg_item, resolve=True)
+        elif isinstance(cfg_item, DictConfig):
+            return {k: to_plain_list(v) for k, v in cfg_item.items()}
+        else:
+            return cfg_item
+
+    kwargs = to_plain_list(cfg)
+    scp_file = kwargs["scp_file"]
+    transcript_file = kwargs["transcript_file"]
+    max_workers = kwargs.get("max_workers", os.cpu_count())
+    jsonl_file = kwargs["jsonl_file"]
+
+    with open(scp_file, "r") as f1, open(transcript_file, "r") as f2:
+        scp_lines = f1.readlines()
+        transcript_lines = f2.readlines()
+
+    if len(scp_lines) != len(transcript_lines):
+        print(f"Warning: Line count mismatch - scp: {len(scp_lines)}, transcript: {len(transcript_lines)}")
+
+    tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
+    processor = LineProcessor(tokenizer)
+
+    data_pairs = list(zip(scp_lines, transcript_lines))
+
+    processed_count = 0
+    failed_count = 0
+    error_messages = []
+
+    with tqdm(total=len(data_pairs), desc="Processing") as pbar:
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            with open(jsonl_file, "w") as f_out:
+                futures = {executor.submit(processor.process_line, pair): i for i, pair in enumerate(data_pairs)}
+
+                for future in as_completed(futures):
+                    result = future.result()
+
+                    if result and "success" in result:
+                        with processor.lock:
+                            json.dump(result["success"], f_out, ensure_ascii=False)
+                            f_out.write("\n")
+                        processed_count += 1
+                    elif result and "error" in result:
+                        failed_count += 1
+                        error_messages.append(result["error"])
+
+                    pbar.update(1)
+                    pbar.set_postfix({"processed": processed_count, "failed": failed_count})
+
+    print(f"\nProcessing completed:")
+    print(f"  Total lines: {len(data_pairs)}")
+    print(f"  Successfully processed: {processed_count}")
+    print(f"  Failed: {failed_count}")
+
+    if error_messages and len(error_messages) <= 10:
+        print(f"\nSample errors:")
+        for error in error_messages[:10]:
+            print(f"  - {error}")
+    elif error_messages:
+        print(f"\nFirst 10 errors:")
+        for error in error_messages[:10]:
+            print(f"  - {error}")
+        print(f"  ... and {len(error_messages) - 10} more errors")
+
+
+if __name__ == "__main__":
+    main_hydra()

funasr/models/fun_asr_nano/tools/utils.py

@@ -0,0 +1,72 @@
+from itertools import groupby
+
+import soundfile as sf
+import torch
+import torchaudio
+import torchaudio.functional as F
+
+
+def load_audio(wav_path, rate: int = None, offset: float = 0, duration: float = None):
+    """Load audio.
+    
+        Args:
+            wav_path: TODO.
+            rate: TODO.
+            offset: TODO.
+            duration: TODO.
+        """
+    with sf.SoundFile(wav_path) as f:
+        start_frame = int(offset * f.samplerate)
+        if duration is None:
+            frames_to_read = f.frames - start_frame
+        else:
+            frames_to_read = int(duration * f.samplerate)
+        f.seek(start_frame)
+        audio_data = f.read(frames_to_read, dtype="float32")
+    audio_tensor = torch.from_numpy(audio_data)
+    if rate is not None and f.samplerate != rate:
+        if audio_tensor.ndim == 1:
+            audio_tensor = audio_tensor.unsqueeze(0)
+        else:
+            audio_tensor = audio_tensor.T
+        resampler = torchaudio.transforms.Resample(orig_freq=f.samplerate, new_freq=rate)
+        audio_tensor = resampler(audio_tensor)
+        if audio_tensor.shape[0] == 1:
+            audio_tensor = audio_tensor.squeeze(0)
+    return audio_tensor, rate if rate is not None else f.samplerate
+
+
+def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0):
+    """Forced align.
+    
+        Args:
+            log_probs: TODO.
+            targets: TODO.
+            blank: TODO.
+        """
+    items = []
+    try:
+        # The current version only supports batch_size==1.
+        log_probs, targets = log_probs.unsqueeze(0).cpu(), targets.unsqueeze(0).cpu()
+        assert log_probs.shape[1] >= targets.shape[1]
+        alignments, scores = F.forced_align(log_probs, targets, blank=blank)
+        alignments, scores = alignments[0], torch.exp(scores[0]).tolist()
+        # use enumerate to keep track of the original indices, then group by token value
+        for token, group in groupby(enumerate(alignments), key=lambda item: item[1]):
+            if token == blank:
+                continue
+            group = list(group)
+            start = group[0][0]
+            end = start + len(group)
+            score = max(scores[start:end])
+            items.append(
+                {
+                    "token": token.item(),
+                    "start_time": start,
+                    "end_time": end,
+                    "score": round(score, 3),
+                }
+            )
+    except:
+        pass
+    return items

funasr/models/fun_asr_nano/tools/whisper_mix_normalize.py

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/python
+# Author: Mengze Chen
+
+import re
+import sys
+
+import cn_tn as cn_tn
+import format5res as cn_itn
+import pyopenjtalk
+import zhconv
+from whisper_normalizer.basic import BasicTextNormalizer
+from whisper_normalizer.english import EnglishTextNormalizer
+
+basic_normalizer = BasicTextNormalizer()
+english_normalizer = EnglishTextNormalizer()
+
+
+def is_only_chinese_and_english(s):
+    # 定义正则表达式模式，匹配中文字符范围和英文字母（包括大小写）
+    """Is only chinese and english.
+    
+        Args:
+            s: TODO.
+        """
+    pattern = r"^[\u4e00-\u9fa5A-Za-z0-9,\.!\?:;，。！？：；、%\'\s\-\~]+$"
+    # 使用正则表达式进行匹配
+    return re.match(pattern, s) is not None
+
+
+def is_only_english(s):
+    # 定义正则表达式模式，匹配中文字符范围和英文字母（包括大小写）
+    """Is only english.
+    
+        Args:
+            s: TODO.
+        """
+    pattern = r"^[A-Za-z0-9,\.!\?:;，。！？：；、%\'\s\-\~]+$"
+    # 使用正则表达式进行匹配
+    return re.match(pattern, s) is not None
+
+
+def is_number(s):
+    # 定义正则表达式模式，匹配中文字符范围和英文字母（包括大小写）
+    """Is number.
+    
+        Args:
+            s: TODO.
+        """
+    pattern = r"^[0-9,\.!\?:;，。！？：；、%\'\s]+$"
+    # 使用正则表达式进行匹配
+    return re.match(pattern, s) is not None
+
+
+def safe_ja_g2p(text, kana=True, max_length=100):
+    """Safe ja g2p.
+    
+        Args:
+            text: Text tensor or string input.
+            kana: TODO.
+            max_length: TODO.
+        """
+    if len(text) > max_length:
+        # 如果文本过长，分段处理
+        parts = []
+        for i in range(0, len(text), max_length):
+            part = text[i : i + max_length]
+            try:
+                converted = pyopenjtalk.g2p(part, kana=kana)
+                parts.append(converted)
+            except:
+                parts.append(part)  # 如果转换失败，使用原文本
+        return " ".join(parts)
+    else:
+        try:
+            return pyopenjtalk.g2p(text, kana=kana)
+        except:
+            return text  # 如果转换失败，返回原文本
+
+
+def normalize_text(srcfn, dstfn, kana=False):
+    """Normalize text.
+    
+        Args:
+            srcfn: TODO.
+            dstfn: TODO.
+            kana: TODO.
+        """
+    with open(srcfn, "r") as f_read, open(dstfn, "w") as f_write:
+        all_lines = f_read.readlines()
+        for line in all_lines:
+            line = line.strip()
+            line_arr = line.split(maxsplit=1)
+            if len(line_arr) < 1:
+                continue
+            if len(line_arr) == 1:
+                line_arr.append("")
+            key = line_arr[0]
+            line_arr[1] = re.sub(r"=", " ", line_arr[1])
+            line_arr[1] = re.sub(r"\(", " ", line_arr[1])
+            line_arr[1] = re.sub(r"\)", " ", line_arr[1])
+            # From Chongjia Ni
+            if kana:
+                line_arr[1] = safe_ja_g2p(line_arr[1], kana=True, max_length=100)
+
+            line_arr = f"{key}\t{line_arr[1]}".split()
+            conts = []
+            language_bak = ""
+            part = []
+            for i in range(1, len(line_arr)):
+                out_part = ""
+                chn_eng_bool = is_only_chinese_and_english(line_arr[i])
+                eng_bool = is_only_english(line_arr[i])
+                num_bool = is_number(line_arr[i])
+                if eng_bool and not num_bool:
+                    language = "en"
+                elif chn_eng_bool:
+                    language = "chn_en"
+                else:
+                    language = "not_chn_en"
+                if language == language_bak or language_bak == "":
+                    part.append(line_arr[i])
+                    language_bak = language
+                else:
+                    if language_bak == "en":
+                        out_part1 = english_normalizer(" ".join(part))
+                        out_part = cn_itn.scoreformat("", out_part1)
+                    elif language_bak == "chn_en":
+                        out_part1 = english_normalizer(" ".join(part))
+                        out_part2 = cn_tn.normalize_nsw(out_part1)
+                        out_part3 = cn_itn.all_convert(out_part2)
+                        out_part = zhconv.convert(out_part3, "zh-cn")
+                    else:
+                        out_part1 = basic_normalizer(" ".join(part))
+                        out_part2 = cn_tn.normalize_nsw(out_part1)
+                        out_part3 = cn_itn.all_convert(out_part2)
+                        out_part = zhconv.convert(out_part3, "zh-cn")
+                    conts.append(out_part)
+                    language_bak = language
+                    part = []
+                    part.append(line_arr[i])
+                if i == len(line_arr) - 1:
+                    if language == "en":
+                        out_part1 = english_normalizer(" ".join(part))
+                        out_part = cn_itn.scoreformat("", out_part1)
+                    elif language == "chn_en":
+                        out_part1 = english_normalizer(" ".join(part))
+                        out_part2 = cn_tn.normalize_nsw(out_part1)
+                        out_part3 = cn_itn.all_convert(out_part2)
+                        out_part = zhconv.convert(out_part3, "zh-cn")
+                    else:
+                        out_part1 = basic_normalizer(" ".join(part))
+                        out_part2 = cn_tn.normalize_nsw(out_part1)
+                        out_part3 = cn_itn.all_convert(out_part2)
+                        out_part = zhconv.convert(out_part3, "zh-cn")
+                    conts.append(out_part)
+
+            f_write.write("{0}\t{1}\n".format(key, " ".join(conts).strip()))
+
+
+if __name__ == "__main__":
+    srcfn = sys.argv[1]
+    dstfn = sys.argv[2]
+    normalize_text(srcfn, dstfn, True if len(sys.argv) > 3 else False)

funasr/models/fun_asr_nano/vllm_utils.py

@@ -0,0 +1,58 @@
+"""Helpers shared by the Fun-ASR-Nano vLLM serving paths.
+
+Kept dependency-free (standard library only) so it can be imported and unit
+tested without a CUDA device or a vLLM installation.
+"""
+
+import logging
+
+logger = logging.getLogger("funasr.fun_asr_nano.vllm")
+
+# A repetition penalty of 1.0 is the identity value, i.e. "no penalty".
+NEUTRAL_REPETITION_PENALTY = 1.0
+
+# Warn only once per process so streaming/batch loops do not spam the log.
+_warned_prompt_embeds = False
+
+
+def resolve_repetition_penalty(repetition_penalty, *, prompt_embeds=True):
+    """Return a repetition penalty that is safe for the requested vLLM mode.
+
+    Fun-ASR-Nano feeds vLLM precomputed audio/text *embeddings* with
+    ``enable_prompt_embeds=True``. In that mode a request carries no prompt
+    token IDs. vLLM applies ``repetition_penalty`` by scattering over the
+    prompt's token IDs, so any value other than 1.0 indexes an empty token-id
+    tensor and aborts the engine with a CUDA
+    ``scatter gather kernel index out of bounds`` assertion (issue #2948).
+
+    When ``prompt_embeds`` is True we therefore force the penalty back to the
+    neutral value and warn once. With ``prompt_embeds=False`` (regular
+    token-prompt decoding) the requested value is passed through unchanged.
+
+    Args:
+        repetition_penalty: Penalty requested by the caller. ``None`` is
+            treated as "unset" and maps to the neutral value.
+        prompt_embeds: Whether the request runs in vLLM prompt-embeds mode.
+
+    Returns:
+        A repetition penalty that will not crash the engine.
+    """
+    global _warned_prompt_embeds
+
+    if repetition_penalty is None:
+        return NEUTRAL_REPETITION_PENALTY
+
+    if prompt_embeds and repetition_penalty != NEUTRAL_REPETITION_PENALTY:
+        if not _warned_prompt_embeds:
+            logger.warning(
+                "repetition_penalty=%s is not supported in vLLM prompt-embeds "
+                "mode (no prompt token IDs to penalize) and would trigger a CUDA "
+                "scatter index-out-of-bounds crash; using repetition_penalty=%s "
+                "instead. See https://github.com/modelscope/FunASR/issues/2948.",
+                repetition_penalty,
+                NEUTRAL_REPETITION_PENALTY,
+            )
+            _warned_prompt_embeds = True
+        return NEUTRAL_REPETITION_PENALTY
+
+    return repetition_penalty