// funasr-sensevoice: SenseVoiceSmall (SAN-M encoder + CTC) on ggml. // fbank.bin (T x 560) -> CMVN -> prepend 4 query tokens -> SAN-M encoder -> // CTC head -> greedy CTC decode -> token ids (stdout). // The encoder is the same SAN-M arch as Fun-ASR-Nano (shared forward). // Detokenize the printed ids with the SentencePiece bpe model (Python side for now). #include "ggml.h" #include "ggml-cpu.h" #include "ggml-alloc.h" #include "ggml-backend.h" #include "gguf.h" #include #include #include #include #include #include #include static const float LN_EPS = 1e-5f; // ---- audio loader: any wav/mp3/flac, any rate/channels -> 16k mono (miniaudio) ---- #define FUNASR_AUDIO_IMPLEMENTATION #include "funasr_audio.h" #include "funasr_vad.h" // built-in FSMN-VAD front end (--vad segmentation) #include static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80,LFR_M=7,LFR_N=6; static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f; static inline float melf(float f){return 1127.0f*logf(1.0f+f/700.0f);} static void fftc(std::vector&re,std::vector&im,int n){ for(int i=1,j=0;i>1;for(;j&b;b>>=1)j^=b;j^=b;if(i compute_fbank(std::vector wav,int&T_out){ for(auto&v:wav)v*=32768.0f; std::vector win(WINLEN); for(int i=0;i> fb(NMEL,std::vector(NBIN,0.0f)); for(int m=0;mL&&mf> feat(T,std::vector(NMEL)); std::vector re(NFFT),im(NFFT),fr(WINLEN); const float fl=1.1920929e-07f; for(int t=0;t0;i--)fr[i]-=PREEMPH*fr[i-1]; fr[0]-=PREEMPH*fr[0]; for(int i=0;i0)e+=fb[m][k]*(re[k]*re[k]+im[k]*im[k]); feat[t][m]=logf(e>fl?e:fl);}} const int pad=(LFR_M-1)/2; int Tl=(T+LFR_N-1)/LFR_N; std::vector> pd; pd.reserve(T+pad+LFR_M); for(int i=0;i out((size_t)Tl*D); for(int i=0;i t; ggml_tensor* g(const std::string&n){auto it=t.find(n);if(it==t.end()){fprintf(stderr,"missing %s\n",n.c_str());exit(1);}return it->second;} }; static ggml_tensor* lin(ggml_context*c,ggml_tensor*w,ggml_tensor*b,ggml_tensor*x){auto y=ggml_mul_mat(c,w,x);return b?ggml_add(c,y,b):y;} static ggml_tensor* lnorm(ggml_context*c,ggml_tensor*x,ggml_tensor*g,ggml_tensor*b){return ggml_add(c,ggml_mul(c,ggml_norm(c,x,LN_EPS),g),b);} static ggml_tensor* sanm_attn(ggml_context*c,model&m,const std::string&p,ggml_tensor*x,int T){ const int D=m.c.d_model,H=m.c.n_head,dk=D/H,K=m.c.kernel; ggml_tensor*qkv=lin(c,m.g(p+"linear_q_k_v.weight"),m.g(p+"linear_q_k_v.bias"),x); size_t nb1=qkv->nb[1]; ggml_tensor*q=ggml_cont(c,ggml_view_2d(c,qkv,D,T,nb1,0)); ggml_tensor*k=ggml_cont(c,ggml_view_2d(c,qkv,D,T,nb1,(size_t)D*sizeof(float))); ggml_tensor*v=ggml_cont(c,ggml_view_2d(c,qkv,D,T,nb1,(size_t)2*D*sizeof(float))); const int pad=(K-1)/2; ggml_tensor*fk=m.g(p+"fsmn_block.weight"); ggml_tensor*vp=ggml_pad_ext(c,v,0,0,pad,pad,0,0,0,0); ggml_tensor*fsmn=v; for(int j=0;jnb[1],(size_t)j*vp->nb[1]); auto wj=ggml_view_1d(c,fk,D,(size_t)j*fk->nb[1]); fsmn=ggml_add(c,fsmn,ggml_mul(c,ggml_cont(c,sl),wj));} q=ggml_permute(c,ggml_reshape_3d(c,q,dk,H,T),0,2,1,3); k=ggml_permute(c,ggml_reshape_3d(c,k,dk,H,T),0,2,1,3); ggml_tensor*vh=ggml_cont(c,ggml_permute(c,ggml_reshape_3d(c,v,dk,H,T),1,2,0,3)); ggml_tensor*kq=ggml_soft_max(c,ggml_scale(c,ggml_mul_mat(c,k,q),1.0f/sqrtf((float)dk))); ggml_tensor*o=ggml_cont_2d(c,ggml_permute(c,ggml_mul_mat(c,vh,kq),0,2,1,3),D,T); return ggml_add(c,lin(c,m.g(p+"linear_out.weight"),m.g(p+"linear_out.bias"),o),fsmn); } static ggml_tensor* sanm_layer(ggml_context*c,model&m,const std::string&p,ggml_tensor*x,int T,bool res){ auto r=x; auto h=lnorm(c,x,m.g(p+"norm1.weight"),m.g(p+"norm1.bias")); auto sa=sanm_attn(c,m,p+"self_attn.",h,T); x=res?ggml_add(c,r,sa):sa; r=x; h=lnorm(c,x,m.g(p+"norm2.weight"),m.g(p+"norm2.bias")); h=lin(c,m.g(p+"feed_forward.w_1.weight"),m.g(p+"feed_forward.w_1.bias"),h); h=ggml_relu(c,h); h=lin(c,m.g(p+"feed_forward.w_2.weight"),m.g(p+"feed_forward.w_2.bias"),h); return ggml_add(c,r,h); } static void add_posenc(std::vector&x,int T,int depth){ double inc=log(10000.0)/(depth/2.0-1.0); for(int t=0;t join, // "▁"(U+2581)->space; meta tokens <|lang|>/<|emo|>/<|event|>/<|itn|> dropped unless --keep-tags. static std::string sv_trim(const std::string&s){size_t a=s.find_first_not_of(' ');if(a==std::string::npos)return "";size_t b=s.find_last_not_of(' ');return s.substr(a,b-a+1);} static std::string detok_sv(const std::vector&ids,const std::vector&vocab,bool keep_tags){ std::string s; for(int id:ids){ if(id<0||id>=(int)vocab.size())continue; const std::string&p=vocab[id]; if(!keep_tags && p.size()>=2 && p[0]=='<' && p[1]=='|') continue; // skip <|...|> meta s+=p; } const std::string lb="\xe2\x96\x81"; size_t pp; while((pp=s.find(lb))!=std::string::npos)s.replace(pp,3," "); return sv_trim(s); } int main(int argc,char**argv){ std::string gguf_path,fbank_path,wav_path,vad_path; int vad_maxseg=30000; bool ids_mode=false,keep_tags=false; for(int i=1;i qtok(nq); for(int i=0;i vocab; {int ki=gguf_find_key(gg,"sv.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i. So no CMVN here. float*emb=(float*)m.g("embed.weight")->data; // [16, 560] row-major // Run encoder+CTC on one fbank window [T,F]; prints greedy-CTC token IDs (no newline). auto run_seg=[&](const std::vector& fb,int T){ int N=nq+T; std::vector inp((size_t)N*F); for(int i=0;i lg((size_t)V*N); ggml_backend_tensor_get(logits,lg.data(),0,ggml_nbytes(logits)); std::vector seg_ids; int prev=-1; // greedy CTC: argmax per frame -> collapse -> drop blank for(int n=0;nbest){best=col[v];am=v;} if(am!=prev && am!=m.c.blank) seg_ids.push_back(am); prev=am; } if(emit_ids){ for(int id:seg_ids) printf("%d ",id); } else { std::string t=detok_sv(seg_ids,vocab,keep_tags); printf("%s",t.c_str()); } ggml_gallocr_free(ga); ggml_free(c); ggml_backend_free(be); }; int64_t t0=ggml_time_us(); if(!vad_path.empty()){ std::vector wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;} std::vector> segs; if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;} for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000); if(end>(int)wav.size())end=wav.size(); if(end-off seg(wav.begin()+off,wav.begin()+end); int t=0; auto fb=compute_fbank(seg,t); run_seg(fb,t); } fprintf(stderr,"[sensevoice] %zu vad segments\n",segs.size()); } else { int32_t T=0,Fc=F; std::vector fb; if(!wav_path.empty()){ std::vector wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;} int t=0; fb=compute_fbank(wav,t); T=t; } else { FILE*f=fopen(fbank_path.c_str(),"rb"); if(!f){fprintf(stderr,"open fbank\n");return 1;} if(fread(&T,4,1,f)!=1||fread(&Fc,4,1,f)!=1){fclose(f);return 1;} fb.resize((size_t)T*Fc); if((int)fread(fb.data(),4,fb.size(),f)!=(int)fb.size()){fclose(f);return 1;} fclose(f); } run_seg(fb,T); } printf("\n"); fprintf(stderr,"[sensevoice] done %.2fs\n",(ggml_time_us()-t0)/1e6); if(m.ctx_w) ggml_free(m.ctx_w); return 0; }