Initial commit: FunASR Speech Recognition Toolkit

Add complete FunASR codebase including models, runtime, and documentation.

funasr/tokenizer/hf_tokenizer.py

@@ -0,0 +1,22 @@
+from funasr.register import tables
+
+
+@tables.register("tokenizer_classes", "HuggingfaceTokenizer")
+def HuggingfaceTokenizer(init_param_path, **kwargs):
+    """Huggingfacetokenizer.
+    
+        Args:
+            init_param_path: TODO.
+            **kwargs: Additional keyword arguments.
+        """
+    try:
+        from transformers import AutoTokenizer
+    except Exception as e:
+        raise ImportError(
+            "HuggingfaceTokenizer requires 'transformers'. "
+            "Please install it with: pip install -U transformers"
+        ) from e
+
+    tokenizer = AutoTokenizer.from_pretrained(init_param_path)
+
+    return tokenizer