你是否在寻找用Python实现语言检测（识别文本语种）的案例

访客自然语言处理 2026-06-04 20:58:48 1

本文目录导读：

方法一：使用 langdetect（推荐，简单易用）
方法二：使用 fastText（高精度）
方法三：使用 spaCy（适合NLP流程）
方法四：使用 transformers（最先进）
方法五：完整的语言检测工具类
性能比较和建议

我来为你提供几个Python实现语言检测的实用案例：

使用 langdetect（推荐，简单易用）

# 安装：pip install langdetect
from langdetect import detect, detect_langs
# 基本使用
texts = [
    "Hello, how are you?",
    "Bonjour, comment allez-vous?",
    "你好，最近怎么样？",
    "こんにちは、お元気ですか？",
    "Hola, ¿cómo estás?"
]
for text in texts:
    lang = detect(text)
    print(f"文本: {text[:30]}... 语言: {lang}")
# 获取语言概率
text = "Hello world, bonjour le monde"
results = detect_langs(text)
print(f"\n语言概率分布:")
for result in results:
    print(f"  语言: {result.lang}, 概率: {result.prob:.2%}")

使用 fastText（高精度）

# 安装：pip install fasttext
# 需要下载语言识别模型：https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz
import fasttext
# 加载预训练模型
model = fasttext.load_model('lid.176.ftz')
def detect_language_fasttext(text):
    predictions = model.predict(text, k=3)
    labels, probabilities = predictions
    print(f"文本: {text[:50]}...")
    for label, prob in zip(labels, probabilities):
        lang_code = label.replace('__label__', '')
        print(f"  语言: {lang_code}, 概率: {prob:.2%}")
# 使用示例
texts = [
    "人工智能正在改变世界",
    "Machine learning is fascinating",
    "L'apprentissage automatique est passionnant"
]
for text in texts:
    detect_language_fasttext(text)
    print()

使用 spaCy（适合NLP流程）

# 安装：pip install spacy spacy-langdetect
# python -m spacy download en_core_web_sm
import spacy
from spacy.language import Language
from spacy_langdetect import LanguageDetector
@Language.factory("language_detector")
def create_language_detector(nlp, name):
    return LanguageDetector()
# 创建NLP对象并添加语言检测组件
nlp = spacy.load("en_core_web_sm")
nlp.add_pipe("language_detector")
def detect_with_spacy(text):
    doc = nlp(text)
    result = doc._.language
    return result['language'], result['score']
# 批量检测
texts = [
    "Python is a great programming language",
    "Python est un excellent langage de programmation",
    "Python是一种优秀的编程语言"
]
for text in texts:
    lang, score = detect_with_spacy(text)
    print(f"文本: {text[:30]}... 语言: {lang}, 置信度: {score:.2%}")

使用 transformers（最先进）

# 安装：pip install transformers torch
from transformers import pipeline
# 加载语言检测pipeline
classifier = pipeline("text-classification", 
                     model="papluca/xlm-roberta-base-language-detection",
                     device=-1)  # 使用CPU
def detect_with_transformers(text, top_k=3):
    results = classifier(text, top_k=top_k)
    print(f"文本: {text[:50]}...")
    for result in results:
        lang = result['label'].split('_')[-1]
        score = result['score']
        print(f"  语言: {lang}, 置信度: {score:.2%}")
# 使用示例
texts = [
    "深度学习正在快速发展",
    "Deep learning is evolving rapidly",
    "深度学习は急速に発展している"
]
for text in texts:
    detect_with_transformers(text)
    print()

完整的语言检测工具类

from typing import List, Tuple, Optional
import json
class LanguageDetector:
    """统一的语言检测工具类"""
    def __init__(self, method='langdetect'):
        self.method = method
        self._setup_detector()
    def _setup_detector(self):
        if self.method == 'langdetect':
            from langdetect import detect, detect_langs
            self.detect = detect
            self.detect_langs = detect_langs
        elif self.method == 'fasttext':
            import fasttext
            self.model = fasttext.load_model('lid.176.ftz')
    def detect(self, text: str) -> str:
        """检测语言"""
        if self.method == 'langdetect':
            return self.detect(text)
        elif self.method == 'fasttext':
            labels, _ = self.model.predict(text)
            return labels[0].replace('__label__', '')
    def detect_with_confidence(self, text: str) -> List[Tuple[str, float]]:
        """检测语言并返回置信度"""
        if self.method == 'langdetect':
            results = self.detect_langs(text)
            return [(r.lang, r.prob) for r in results]
        elif self.method == 'fasttext':
            labels, probs = self.model.predict(text, k=3)
            return [(l.replace('__label__', ''), p) for l, p in zip(labels, probs)]
# 使用示例
detector = LanguageDetector(method='langdetect')
# 批量检测文件中的文本
def detect_file_languages(file_path: str):
    with open(file_path, 'r', encoding='utf-8') as f:
        lines = f.readlines()
    results = []
    for line in lines:
        line = line.strip()
        if line:
            lang = detector.detect(line)
            results.append({'text': line, 'language': lang})
    # 统计语言分布
    lang_distribution = {}
    for r in results:
        lang = r['language']
        lang_distribution[lang] = lang_distribution.get(lang, 0) + 1
    return results, lang_distribution
# 保存检测结果到JSON
results, distribution = detect_file_languages('sample_texts.txt')
print("语言分布:", json.dumps(distribution, indent=2))
print("检测结果:")
for r in results[:5]:  # 显示前5条
    print(f"  {r['text'][:40]} -> {r['language']}")

性能比较和建议

import time
# 性能测试函数
def benchmark_detectors(texts: List[str]):
    results = {}
    # 测试 langdetect
    from langdetect import detect
    start = time.time()
    for text in texts:
        detect(text)
    results['langdetect'] = time.time() - start
    # 测试 fastText（如果已安装）
    try:
        import fasttext
        model = fasttext.load_model('lid.176.ftz')
        start = time.time()
        for text in texts:
            model.predict(text)
        results['fasttext'] = time.time() - start
    except:
        pass
    return results
# 使用建议
recommendations = """
推荐方案:
1. 简单需求: langdetect - 易于使用，精度尚可
2. 生产环境: fastText - 速度快，精度高
3. NLP流程: spaCy - 与NLP生态系统集成
4. 最高精度: transformers - 最先进，但速度较慢
5. 多语言混合: 使用多个检测器ensemble
"""
print(recommendations)

选择建议：