Initial commit: FunASR Speech Recognition Toolkit
Update API Documentation / build-api-docs (push) Has been cancelled
Update API Documentation / build-api-docs (push) Has been cancelled
Add complete FunASR codebase including models, runtime, and documentation.
This commit is contained in:
freedakgmail
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,299 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#!/usr/bin/python
|
||||
# Author: Mengze Chen
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
def scoreformat(name, line, flag=1):
|
||||
newline = ""
|
||||
for i in range(0, len(line)):
|
||||
curr = line[i]
|
||||
currEn = False
|
||||
if curr == "":
|
||||
continue
|
||||
if (
|
||||
(curr >= "\u0041" and curr <= "\u005a") # eng
|
||||
or (curr >= "\u0061" and curr <= "\u007a") # eng
|
||||
or (curr >= "\u0000" and curr <= "\u007f") # de fr es it
|
||||
or (curr >= "\u0400" and curr <= "\u04ff") # ru
|
||||
or (curr >= "\u0100" and curr <= "\u017f") # latin1
|
||||
or (curr >= "\u0080" and curr <= "\u00ff") # latin2
|
||||
or curr == "'"
|
||||
) and (curr < "\u0030" or curr > "\u0039"):
|
||||
currEn = True
|
||||
if i == 0:
|
||||
newline = newline + curr
|
||||
else:
|
||||
if lastEn == True and currEn == True:
|
||||
newline = newline + curr
|
||||
else:
|
||||
newline = newline + " " + curr
|
||||
if flag == -1:
|
||||
lastEn = False
|
||||
else:
|
||||
lastEn = currEn
|
||||
ret = re.sub("[ ]{1,}", " ", newline)
|
||||
ret = ret
|
||||
if name == "":
|
||||
ret = ret
|
||||
else:
|
||||
if flag <= 0:
|
||||
ret = ret + " " + "(" + name + ")"
|
||||
else:
|
||||
ret = name + "\t" + ret
|
||||
return ret
|
||||
|
||||
|
||||
def recoformat(line):
|
||||
newline = ""
|
||||
en_flag = 0 # 0: no-english 1 : english 2: former
|
||||
for i in range(0, len(line)):
|
||||
word = line[i]
|
||||
if ord(word) == 32:
|
||||
if en_flag == 0:
|
||||
continue
|
||||
else:
|
||||
en_flag = 0
|
||||
newline += " "
|
||||
if (word >= "\u4e00" and word <= "\u9fa5") or (word >= "\u0030" and word <= "\u0039"):
|
||||
if en_flag == 1:
|
||||
newline += " " + word
|
||||
else:
|
||||
newline += word
|
||||
en_flag = 0
|
||||
elif (
|
||||
(word >= "\u0041" and word <= "\u005a") # eng
|
||||
or (word >= "\u0061" and word <= "\u007a") # eng
|
||||
or (word >= "\u0000" and word <= "\u007f") # de fr es it
|
||||
or (word >= "\u0400" and word <= "\u04ff") # ru
|
||||
or (word >= "\u0100" and word <= "\u017f") # latin1
|
||||
or (word >= "\u0080" and word <= "\u00ff") # latin2
|
||||
or word == "'"
|
||||
):
|
||||
if en_flag == 0:
|
||||
newline += " " + ("" if (word == "'") else word)
|
||||
else:
|
||||
newline += word
|
||||
en_flag = 1
|
||||
else:
|
||||
newline += " " + word
|
||||
newline = newline
|
||||
newline = re.sub("[ ]{1,}", " ", newline)
|
||||
newline = newline
|
||||
return newline
|
||||
|
||||
|
||||
def numbersingle(line):
|
||||
chnu = ["零", "一", "二", "两", "三", "四", "五", "六", "七", "八", "九", "点"]
|
||||
newline = ""
|
||||
for id in range(len(line)):
|
||||
if re.findall(r"\.", line[id]):
|
||||
if re.findall(r"\.\s*$", line[id]):
|
||||
newline += "."
|
||||
else:
|
||||
newline += chnu[10]
|
||||
elif re.search(r"0", line[id]):
|
||||
if id > 0 and id < len(line) - 1:
|
||||
if (
|
||||
re.search(r"\d", line[id - 1])
|
||||
and (not re.search(r"\d", line[id + 1]))
|
||||
and (not re.search(r"0", line[id - 1]))
|
||||
):
|
||||
if id > 2 and len(line) > 2 and (not re.search(r"\d", line[id - 1])):
|
||||
newline = newline[:-1]
|
||||
newline += chnu[int(line[id - 1])] + "十"
|
||||
else:
|
||||
newline += chnu[int(line[id])]
|
||||
else:
|
||||
newline += chnu[int(line[id])]
|
||||
else:
|
||||
newline += chnu[int(line[id])]
|
||||
elif re.search(r"\d", line[id]):
|
||||
newline += chnu[int(line[id])]
|
||||
else:
|
||||
newline += line[id]
|
||||
return newline
|
||||
|
||||
|
||||
def ch_number2digit(line):
|
||||
number_flag = 0
|
||||
zero_flag = 0
|
||||
bits = {
|
||||
"零": "1",
|
||||
"十": "2",
|
||||
"百": "3",
|
||||
"千": "4",
|
||||
"万": "5",
|
||||
"十万": "6",
|
||||
"百万": "7",
|
||||
"千万": "8",
|
||||
}
|
||||
chsh = {
|
||||
"一": "1",
|
||||
"二": "2",
|
||||
"三": "3",
|
||||
"四": "4",
|
||||
"五": "5",
|
||||
"六": "6",
|
||||
"七": "7",
|
||||
"八": "8",
|
||||
"九": "9",
|
||||
"两": "2",
|
||||
"幺": "1",
|
||||
}
|
||||
unit = {"里": "1", "克": "1", "米": "1"}
|
||||
newline = ""
|
||||
digit = []
|
||||
bit = []
|
||||
onebit = ""
|
||||
for i in range(len(line)):
|
||||
if ord(line[i]) == 32:
|
||||
newline += " "
|
||||
continue
|
||||
if line[i] in chsh:
|
||||
number_flag = 1
|
||||
if line[i] == "两":
|
||||
if (i == len(line) - 1) or ((line[i + 1] not in chsh.keys()) and (line[i + 1] not in bits.keys())):
|
||||
number_flag = -1
|
||||
if number_flag == 1:
|
||||
digit.append(chsh[line[i]])
|
||||
|
||||
elif "十" == line[i] and number_flag == 0:
|
||||
number_flag = 2
|
||||
digit.append("1")
|
||||
bit.append(line[i])
|
||||
elif "十" == line[i] and number_flag == 3:
|
||||
digit.append("1")
|
||||
bit.append(line[i])
|
||||
elif ("零" == line[i]) and (number_flag == 0 or number_flag == 1):
|
||||
digit.append("0")
|
||||
elif ("零" == line[i]) and number_flag == 3:
|
||||
zero_flag = 1
|
||||
elif number_flag == 1 and line[i] in bits:
|
||||
number_flag = 3
|
||||
if line[i] == "千":
|
||||
if i < len(line) - 1:
|
||||
if line[i + 1] in unit:
|
||||
number_flag = -1
|
||||
if number_flag == 3:
|
||||
onebit = line[i]
|
||||
bit.append(onebit)
|
||||
elif number_flag == 3 and line[i] in bits:
|
||||
onebit = bit[-1] + line[i]
|
||||
if onebit in bits:
|
||||
bit[-1] = onebit
|
||||
else:
|
||||
number_flag = -2
|
||||
else:
|
||||
number_flag = -1
|
||||
if len(digit) > 0 and number_flag == -1:
|
||||
number_flag = -2
|
||||
if i == (len(line) - 1) and number_flag >= 0:
|
||||
number_flag = -1
|
||||
if number_flag < 0:
|
||||
newdigit = ""
|
||||
if len(digit) > 0: # and (len(digit) == len(bit))):
|
||||
if len(bit) == 1 and zero_flag == 0 and bit[0] == "百" and len(bit) != len(digit):
|
||||
bit.append("十")
|
||||
if len(digit) == (len(bit) + 1):
|
||||
bit.append("零")
|
||||
if len(digit) == len(bit):
|
||||
for m in range(len(digit))[-1::-1]:
|
||||
if int(bits[bit[m]]) == int(len(newdigit) + 1):
|
||||
newdigit += digit[m]
|
||||
else:
|
||||
nu = int(bits[bit[m]]) - len(newdigit) - 1
|
||||
for n in range(nu):
|
||||
newdigit += "0"
|
||||
newdigit += digit[m]
|
||||
for z in range(len(newdigit))[-1::-1]:
|
||||
newline += newdigit[z]
|
||||
else:
|
||||
newline += "".join(digit)
|
||||
bit = []
|
||||
digit = []
|
||||
zero_flag = 0
|
||||
else:
|
||||
newline += line[i]
|
||||
if number_flag == -2:
|
||||
newline += line[i]
|
||||
number_flag = 0
|
||||
return newline
|
||||
|
||||
|
||||
def special(line):
|
||||
newline = ""
|
||||
for e in range(len(line)):
|
||||
if ord(line[e]) == 247:
|
||||
newline += "除以"
|
||||
elif ord(line[e]) == 215:
|
||||
newline += "乘以"
|
||||
elif ord(line[e]) == 61:
|
||||
newline += "等于"
|
||||
elif ord(line[e]) == 43:
|
||||
newline += "加"
|
||||
elif ord(line[e]) == 45:
|
||||
newline += "负"
|
||||
elif ord(line[e]) == 8451:
|
||||
newline += "摄氏度"
|
||||
elif ord(line[e]) == 13217:
|
||||
newline += "平方米"
|
||||
elif ord(line[e]) == 8240 or ord(line[e]) == 65130:
|
||||
newline += "%"
|
||||
elif ord(line[e]) == 46:
|
||||
newline += "点"
|
||||
elif ord(line[e]) == 176:
|
||||
newline += "度"
|
||||
angel = 1
|
||||
elif ord(line[e]) == 8242 and angel == 1:
|
||||
newline += "分"
|
||||
else:
|
||||
newline += line[e]
|
||||
return newline
|
||||
|
||||
|
||||
def all_convert(content):
|
||||
content = recoformat(content)
|
||||
content = numbersingle(content)
|
||||
content = ch_number2digit(content)
|
||||
content = special(content)
|
||||
content = scoreformat("", content)
|
||||
return content
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv[1:]) < 1:
|
||||
sys.stderr.write("Usage:\n .py reco.result\n")
|
||||
sys.stderr.write(" reco.result: id<tab>recoresult\n")
|
||||
sys.exit(1)
|
||||
f = open(sys.argv[1])
|
||||
flag = 0
|
||||
if len(sys.argv[1:]) > 1:
|
||||
flag = int(sys.argv[2])
|
||||
for line in f.readlines():
|
||||
if not line:
|
||||
continue
|
||||
line = line.rstrip()
|
||||
tmp = line.split("\t")
|
||||
if len(tmp) < 2:
|
||||
tmp = line.split(",")
|
||||
if len(tmp) < 2:
|
||||
tmp = line.split(" ", 1)
|
||||
if len(tmp) < 2:
|
||||
name = tmp[0]
|
||||
content = ""
|
||||
print(content)
|
||||
continue
|
||||
name = tmp[0]
|
||||
content = tmp[1]
|
||||
name = re.sub("\.pcm", "", name)
|
||||
name = re.sub("\.wav", "", name)
|
||||
content = recoformat(content)
|
||||
content = numbersingle(content)
|
||||
content = ch_number2digit(content)
|
||||
content = special(content)
|
||||
content = scoreformat(name, content, flag)
|
||||
print(content)
|
||||
f.close()
|
||||
@@ -0,0 +1,137 @@
|
||||
import hydra
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from io import BytesIO
|
||||
from typing import Dict, Optional, Tuple
|
||||
from urllib.request import urlopen
|
||||
|
||||
import soundfile as sf
|
||||
from modelscope import AutoTokenizer
|
||||
from tqdm import tqdm
|
||||
from omegaconf import DictConfig, OmegaConf, ListConfig
|
||||
|
||||
|
||||
class LineProcessor:
|
||||
def __init__(self, tokenizer):
|
||||
self.tokenizer = tokenizer
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def process_line(self, line_pair: Tuple[str, str]) -> Optional[Dict]:
|
||||
line1, line2 = line_pair
|
||||
|
||||
line1, line2 = line1.strip(), line2.strip()
|
||||
if not line1 or not line2:
|
||||
return None
|
||||
|
||||
parts1, parts2 = line1.split(maxsplit=1), line2.split(maxsplit=1)
|
||||
if len(parts1) != 2 or len(parts2) != 2:
|
||||
return None
|
||||
|
||||
utt1, utt2 = parts1[0], parts2[0]
|
||||
wav_path, text = parts1[1], parts2[1]
|
||||
|
||||
if utt1 != utt2:
|
||||
return {"error": f"UTT mismatch: {utt1} vs {utt2}"}
|
||||
|
||||
try:
|
||||
if wav_path.startswith("http"):
|
||||
response = urlopen(wav_path)
|
||||
if response.status != 200:
|
||||
return {"error": f"WAV not found: {wav_path}"}
|
||||
audio_file = BytesIO(response.read())
|
||||
duration = sf.info(audio_file).duration
|
||||
else:
|
||||
if not os.path.exists(wav_path):
|
||||
return {"error": f"WAV not found: {wav_path}"}
|
||||
duration = sf.info(wav_path).duration
|
||||
|
||||
data = {
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"语音转写：<|startofspeech|>!{wav_path}<|endofspeech|>",
|
||||
},
|
||||
{"role": "assistant", "content": text},
|
||||
],
|
||||
"speech_length": int((duration * 1000 - 25) // 10 + 1),
|
||||
"text_length": len(self.tokenizer.tokenize(text)),
|
||||
}
|
||||
return {"success": data, "utt": utt1}
|
||||
|
||||
except Exception as e:
|
||||
return {"error": f"Error processing {wav_path}: {str(e)}"}
|
||||
|
||||
|
||||
@hydra.main(config_name=None, version_base=None)
|
||||
def main_hydra(cfg: DictConfig):
|
||||
def to_plain_list(cfg_item):
|
||||
if isinstance(cfg_item, ListConfig):
|
||||
return OmegaConf.to_container(cfg_item, resolve=True)
|
||||
elif isinstance(cfg_item, DictConfig):
|
||||
return {k: to_plain_list(v) for k, v in cfg_item.items()}
|
||||
else:
|
||||
return cfg_item
|
||||
|
||||
kwargs = to_plain_list(cfg)
|
||||
scp_file = kwargs["scp_file"]
|
||||
transcript_file = kwargs["transcript_file"]
|
||||
max_workers = kwargs.get("max_workers", os.cpu_count())
|
||||
jsonl_file = kwargs["jsonl_file"]
|
||||
|
||||
with open(scp_file, "r") as f1, open(transcript_file, "r") as f2:
|
||||
scp_lines = f1.readlines()
|
||||
transcript_lines = f2.readlines()
|
||||
|
||||
if len(scp_lines) != len(transcript_lines):
|
||||
print(f"Warning: Line count mismatch - scp: {len(scp_lines)}, transcript: {len(transcript_lines)}")
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
|
||||
processor = LineProcessor(tokenizer)
|
||||
|
||||
data_pairs = list(zip(scp_lines, transcript_lines))
|
||||
|
||||
processed_count = 0
|
||||
failed_count = 0
|
||||
error_messages = []
|
||||
|
||||
with tqdm(total=len(data_pairs), desc="Processing") as pbar:
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
with open(jsonl_file, "w") as f_out:
|
||||
futures = {executor.submit(processor.process_line, pair): i for i, pair in enumerate(data_pairs)}
|
||||
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
|
||||
if result and "success" in result:
|
||||
with processor.lock:
|
||||
json.dump(result["success"], f_out, ensure_ascii=False)
|
||||
f_out.write("\n")
|
||||
processed_count += 1
|
||||
elif result and "error" in result:
|
||||
failed_count += 1
|
||||
error_messages.append(result["error"])
|
||||
|
||||
pbar.update(1)
|
||||
pbar.set_postfix({"processed": processed_count, "failed": failed_count})
|
||||
|
||||
print(f"\nProcessing completed:")
|
||||
print(f" Total lines: {len(data_pairs)}")
|
||||
print(f" Successfully processed: {processed_count}")
|
||||
print(f" Failed: {failed_count}")
|
||||
|
||||
if error_messages and len(error_messages) <= 10:
|
||||
print(f"\nSample errors:")
|
||||
for error in error_messages[:10]:
|
||||
print(f" - {error}")
|
||||
elif error_messages:
|
||||
print(f"\nFirst 10 errors:")
|
||||
for error in error_messages[:10]:
|
||||
print(f" - {error}")
|
||||
print(f" ... and {len(error_messages) - 10} more errors")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main_hydra()
|
||||
@@ -0,0 +1,57 @@
|
||||
from itertools import groupby
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
import torchaudio
|
||||
import torchaudio.functional as F
|
||||
|
||||
|
||||
def load_audio(wav_path, rate: int = None, offset: float = 0, duration: float = None):
|
||||
with sf.SoundFile(wav_path) as f:
|
||||
start_frame = int(offset * f.samplerate)
|
||||
if duration is None:
|
||||
frames_to_read = f.frames - start_frame
|
||||
else:
|
||||
frames_to_read = int(duration * f.samplerate)
|
||||
f.seek(start_frame)
|
||||
audio_data = f.read(frames_to_read, dtype="float32")
|
||||
audio_tensor = torch.from_numpy(audio_data)
|
||||
if rate is not None and f.samplerate != rate:
|
||||
if audio_tensor.ndim == 1:
|
||||
audio_tensor = audio_tensor.unsqueeze(0)
|
||||
else:
|
||||
audio_tensor = audio_tensor.T
|
||||
resampler = torchaudio.transforms.Resample(orig_freq=f.samplerate, new_freq=rate)
|
||||
audio_tensor = resampler(audio_tensor)
|
||||
if audio_tensor.shape[0] == 1:
|
||||
audio_tensor = audio_tensor.squeeze(0)
|
||||
return audio_tensor, rate if rate is not None else f.samplerate
|
||||
|
||||
|
||||
def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0):
|
||||
items = []
|
||||
try:
|
||||
# The current version only supports batch_size==1.
|
||||
log_probs, targets = log_probs.unsqueeze(0).cpu(), targets.unsqueeze(0).cpu()
|
||||
assert log_probs.shape[1] >= targets.shape[1]
|
||||
alignments, scores = F.forced_align(log_probs, targets, blank=blank)
|
||||
alignments, scores = alignments[0], torch.exp(scores[0]).tolist()
|
||||
# use enumerate to keep track of the original indices, then group by token value
|
||||
for token, group in groupby(enumerate(alignments), key=lambda item: item[1]):
|
||||
if token == blank:
|
||||
continue
|
||||
group = list(group)
|
||||
start = group[0][0]
|
||||
end = start + len(group)
|
||||
score = max(scores[start:end])
|
||||
items.append(
|
||||
{
|
||||
"token": token.item(),
|
||||
"start_time": start,
|
||||
"end_time": end,
|
||||
"score": round(score, 3),
|
||||
}
|
||||
)
|
||||
except:
|
||||
pass
|
||||
return items
|
||||
@@ -0,0 +1,135 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#!/usr/bin/python
|
||||
# Author: Mengze Chen
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
import cn_tn as cn_tn
|
||||
import format5res as cn_itn
|
||||
import pyopenjtalk
|
||||
import zhconv
|
||||
from whisper_normalizer.basic import BasicTextNormalizer
|
||||
from whisper_normalizer.english import EnglishTextNormalizer
|
||||
|
||||
basic_normalizer = BasicTextNormalizer()
|
||||
english_normalizer = EnglishTextNormalizer()
|
||||
|
||||
|
||||
def is_only_chinese_and_english(s):
|
||||
# 定义正则表达式模式，匹配中文字符范围和英文字母（包括大小写）
|
||||
pattern = r"^[\u4e00-\u9fa5A-Za-z0-9,\.!\?:;，。！？：；、%\'\s\-\~]+$"
|
||||
# 使用正则表达式进行匹配
|
||||
return re.match(pattern, s) is not None
|
||||
|
||||
|
||||
def is_only_english(s):
|
||||
# 定义正则表达式模式，匹配中文字符范围和英文字母（包括大小写）
|
||||
pattern = r"^[A-Za-z0-9,\.!\?:;，。！？：；、%\'\s\-\~]+$"
|
||||
# 使用正则表达式进行匹配
|
||||
return re.match(pattern, s) is not None
|
||||
|
||||
|
||||
def is_number(s):
|
||||
# 定义正则表达式模式，匹配中文字符范围和英文字母（包括大小写）
|
||||
pattern = r"^[0-9,\.!\?:;，。！？：；、%\'\s]+$"
|
||||
# 使用正则表达式进行匹配
|
||||
return re.match(pattern, s) is not None
|
||||
|
||||
|
||||
def safe_ja_g2p(text, kana=True, max_length=100):
|
||||
if len(text) > max_length:
|
||||
# 如果文本过长，分段处理
|
||||
parts = []
|
||||
for i in range(0, len(text), max_length):
|
||||
part = text[i : i + max_length]
|
||||
try:
|
||||
converted = pyopenjtalk.g2p(part, kana=kana)
|
||||
parts.append(converted)
|
||||
except:
|
||||
parts.append(part) # 如果转换失败，使用原文本
|
||||
return " ".join(parts)
|
||||
else:
|
||||
try:
|
||||
return pyopenjtalk.g2p(text, kana=kana)
|
||||
except:
|
||||
return text # 如果转换失败，返回原文本
|
||||
|
||||
|
||||
def normalize_text(srcfn, dstfn, kana=False):
|
||||
with open(srcfn, "r") as f_read, open(dstfn, "w") as f_write:
|
||||
all_lines = f_read.readlines()
|
||||
for line in all_lines:
|
||||
line = line.strip()
|
||||
line_arr = line.split(maxsplit=1)
|
||||
if len(line_arr) < 1:
|
||||
continue
|
||||
if len(line_arr) == 1:
|
||||
line_arr.append("")
|
||||
key = line_arr[0]
|
||||
line_arr[1] = re.sub(r"=", " ", line_arr[1])
|
||||
line_arr[1] = re.sub(r"\(", " ", line_arr[1])
|
||||
line_arr[1] = re.sub(r"\)", " ", line_arr[1])
|
||||
# From Chongjia Ni
|
||||
if kana:
|
||||
line_arr[1] = safe_ja_g2p(line_arr[1], kana=True, max_length=100)
|
||||
|
||||
line_arr = f"{key}\t{line_arr[1]}".split()
|
||||
conts = []
|
||||
language_bak = ""
|
||||
part = []
|
||||
for i in range(1, len(line_arr)):
|
||||
out_part = ""
|
||||
chn_eng_bool = is_only_chinese_and_english(line_arr[i])
|
||||
eng_bool = is_only_english(line_arr[i])
|
||||
num_bool = is_number(line_arr[i])
|
||||
if eng_bool and not num_bool:
|
||||
language = "en"
|
||||
elif chn_eng_bool:
|
||||
language = "chn_en"
|
||||
else:
|
||||
language = "not_chn_en"
|
||||
if language == language_bak or language_bak == "":
|
||||
part.append(line_arr[i])
|
||||
language_bak = language
|
||||
else:
|
||||
if language_bak == "en":
|
||||
out_part1 = english_normalizer(" ".join(part))
|
||||
out_part = cn_itn.scoreformat("", out_part1)
|
||||
elif language_bak == "chn_en":
|
||||
out_part1 = english_normalizer(" ".join(part))
|
||||
out_part2 = cn_tn.normalize_nsw(out_part1)
|
||||
out_part3 = cn_itn.all_convert(out_part2)
|
||||
out_part = zhconv.convert(out_part3, "zh-cn")
|
||||
else:
|
||||
out_part1 = basic_normalizer(" ".join(part))
|
||||
out_part2 = cn_tn.normalize_nsw(out_part1)
|
||||
out_part3 = cn_itn.all_convert(out_part2)
|
||||
out_part = zhconv.convert(out_part3, "zh-cn")
|
||||
conts.append(out_part)
|
||||
language_bak = language
|
||||
part = []
|
||||
part.append(line_arr[i])
|
||||
if i == len(line_arr) - 1:
|
||||
if language == "en":
|
||||
out_part1 = english_normalizer(" ".join(part))
|
||||
out_part = cn_itn.scoreformat("", out_part1)
|
||||
elif language == "chn_en":
|
||||
out_part1 = english_normalizer(" ".join(part))
|
||||
out_part2 = cn_tn.normalize_nsw(out_part1)
|
||||
out_part3 = cn_itn.all_convert(out_part2)
|
||||
out_part = zhconv.convert(out_part3, "zh-cn")
|
||||
else:
|
||||
out_part1 = basic_normalizer(" ".join(part))
|
||||
out_part2 = cn_tn.normalize_nsw(out_part1)
|
||||
out_part3 = cn_itn.all_convert(out_part2)
|
||||
out_part = zhconv.convert(out_part3, "zh-cn")
|
||||
conts.append(out_part)
|
||||
|
||||
f_write.write("{0}\t{1}\n".format(key, " ".join(conts).strip()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
srcfn = sys.argv[1]
|
||||
dstfn = sys.argv[2]
|
||||
normalize_text(srcfn, dstfn, True if len(sys.argv) > 3 else False)
|
||||
Reference in New Issue
Block a user