100 lines
3.1 KiB
Python
100 lines
3.1 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
文本处理器集合
|
||
每个处理器签名：fn(raw: Any, data: IOData) -> Any
|
||
可自由组合成处理链，由 IODispatcher 顺序调用
|
||
"""
|
||
import logging
|
||
import re
|
||
import time
|
||
|
||
from uas.io.base_io import IOData
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
def processor_clean_text(raw: str, data: IOData) -> str:
|
||
"""清洗文本：去除多余空白、特殊字符"""
|
||
result = re.sub(r'\s+', ' ', str(raw)).strip()
|
||
result = re.sub(r'[^\w\s\u4e00-\u9fff.,!?，。！？、；：""''（）【】]', '', result)
|
||
logger.debug(f"[clean] {raw!r} -> {result!r}")
|
||
return result
|
||
|
||
|
||
def processor_to_upper(raw: str, data: IOData) -> str:
|
||
"""英文转大写"""
|
||
return raw.upper()
|
||
|
||
|
||
def processor_add_timestamp(raw: str, data: IOData) -> str:
|
||
"""追加时间戳"""
|
||
ts = time.strftime("%Y-%m-%d %H:%M:%S")
|
||
return f"[{ts}] {raw}"
|
||
|
||
|
||
def processor_word_count(raw: str, data: IOData) -> str:
|
||
"""统计字数并追加到 metadata"""
|
||
words = len(raw.split())
|
||
chars = len(raw)
|
||
data.metadata["word_count"] = words
|
||
data.metadata["char_count"] = chars
|
||
logger.debug(f"字数统计: words={words} chars={chars}")
|
||
return raw # 不修改内容，仅更新 metadata
|
||
|
||
|
||
def processor_keyword_extract(raw: str, data: IOData) -> str:
|
||
"""简单关键词提取（示例：提取长度>3的词）"""
|
||
words = re.findall(r'\b\w{4,}\b', raw)
|
||
keywords = list(set(words))[:10]
|
||
data.metadata["keywords"] = keywords
|
||
logger.debug(f"关键词: {keywords}")
|
||
return raw
|
||
|
||
|
||
def processor_sensitive_filter(raw: str, data: IOData) -> str:
|
||
"""敏感词过滤（示例）"""
|
||
sensitive = ["badword1", "badword2"]
|
||
result = raw
|
||
for word in sensitive:
|
||
result = result.replace(word, "*" * len(word))
|
||
return result
|
||
|
||
|
||
def processor_simulate_nlp(raw: str, data: IOData) -> dict:
|
||
"""
|
||
模拟 NLP 处理（实际可替换为 transformers / spacy 等）
|
||
返回结构化结果
|
||
"""
|
||
import time
|
||
time.sleep(0.05) # 模拟耗时
|
||
return {
|
||
"original": raw,
|
||
"length": len(raw),
|
||
"summary": raw[:50] + ("..." if len(raw) > 50 else ""),
|
||
"intent": "query" if "?" in raw or "？" in raw else "statement",
|
||
"lang": "zh" if re.search(r'[\u4e00-\u9fff]', raw) else "en",
|
||
}
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────
|
||
# 处理器工厂：预设处理链
|
||
# ─────────────────────────────────────────────────────────────────
|
||
|
||
PIPELINE_PRESETS = {
|
||
"basic": [
|
||
processor_clean_text,
|
||
processor_sensitive_filter,
|
||
processor_word_count,
|
||
],
|
||
"full": [
|
||
processor_clean_text,
|
||
processor_sensitive_filter,
|
||
processor_word_count,
|
||
processor_keyword_extract,
|
||
processor_simulate_nlp,
|
||
],
|
||
"speech": [
|
||
processor_clean_text,
|
||
processor_add_timestamp,
|
||
],
|
||
} |