langid.py

import langid

# Set languages to detect (optional, default is all 97)
langid.set_languages(['en', 'es', 'fr', 'de', 'it'])

# Detect language of a single text
text = "Hello, how are you today?"
lang, confidence = langid.classify(text)
print(f"Language: {lang}, Confidence: {confidence:.4f}")

# Detect multiple texts
texts = [
    "Hello world",
    "Bonjour le monde", 
    "Hola mundo",
    "Hallo Welt"
]

for text in texts:
    lang, conf = langid.classify(text)
    print(f"'{text}' -> {lang} ({conf:.3f})")

# Reset to all languages
langid.set_languages(None)

import langid

def batch_language_detection(texts, confidence_threshold=0.8):
    """Process multiple texts with confidence filtering."""
    results = []
    
    for text in texts:
        if len(text.strip()) < 20:  # Skip very short texts
            results.append(('unknown', 0.0, 'too_short'))
            continue
            
        lang, confidence = langid.classify(text)
        
        if confidence >= confidence_threshold:
            results.append((lang, confidence, 'reliable'))
        else:
            results.append((lang, confidence, 'uncertain'))
    
    return results

# Example usage
web_comments = [
    "This is an excellent product, highly recommended!",
    "Ce produit est vraiment fantastique, je le recommande vivement!",
    "Este producto es excelente, lo recomiendo mucho!",
    "xyz123",  # Too short
    "混合语言 mixed language content here",  # Mixed language
]

results = batch_language_detection(web_comments)
for i, (text, (lang, conf, status)) in enumerate(zip(web_comments, results)):
    print(f"Text {i+1}: {lang} ({conf:.3f}) - {status}")
    
# Filter reliable detections only
reliable_results = [(text, lang) for text, (lang, conf, status) 
                   in zip(web_comments, results) if status == 'reliable']

print(f"\nReliable detections: {len(reliable_results)}/{len(web_comments)}")

from flask import Flask, request, jsonify
import langid
import time

app = Flask(__name__)

# Configure langid for common web languages
COMMON_LANGUAGES = ['en', 'es', 'fr', 'de', 'it', 'pt', 'ru', 'zh', 'ja', 'ko']
langid.set_languages(COMMON_LANGUAGES)

@app.route('/detect-language', methods=['POST'])
def detect_language():
    """API endpoint for language detection."""
    try:
        data = request.get_json()
        text = data.get('text', '').strip()
        
        if not text:
            return jsonify({'error': 'No text provided'}), 400
            
        if len(text) < 10:
            return jsonify({'error': 'Text too short for reliable detection'}), 400
        
        start_time = time.time()
        language, confidence = langid.classify(text)
        processing_time = (time.time() - start_time) * 1000  # Convert to ms
        
        return jsonify({
            'language': language,
            'confidence': round(confidence, 4),
            'processing_time_ms': round(processing_time, 2),
            'text_length': len(text),
            'reliable': confidence >= 0.8
        })
        
    except Exception as e:
        return jsonify({'error': str(e)}), 500

@app.route('/batch-detect', methods=['POST'])
def batch_detect():
    """Batch language detection endpoint."""
    try:
        data = request.get_json()
        texts = data.get('texts', [])
        
        if not isinstance(texts, list):
            return jsonify({'error': 'Texts must be a list'}), 400
            
        if len(texts) > 1000:  # Limit batch size
            return jsonify({'error': 'Batch size too large (max 1000)'}), 400
        
        start_time = time.time()
        results = []
        
        for text in texts:
            if isinstance(text, str) and len(text.strip()) >= 10:
                lang, conf = langid.classify(text.strip())
                results.append({
                    'text': text[:100] + '...' if len(text) > 100 else text,
                    'language': lang,
                    'confidence': round(conf, 4)
                })
            else:
                results.append({
                    'text': text[:100] + '...' if len(str(text)) > 100 else str(text),
                    'language': 'unknown',
                    'confidence': 0.0
                })
        
        total_time = (time.time() - start_time) * 1000
        
        return jsonify({
            'results': results,
            'total_processing_time_ms': round(total_time, 2),
            'texts_processed': len(texts),
            'average_time_per_text_ms': round(total_time / len(texts), 2) if texts else 0
        })
        
    except Exception as e:
        return jsonify({'error': str(e)}), 500

if __name__ == '__main__':
    app.run(debug=True)