100 lines
3.1 KiB
Python
100 lines
3.1 KiB
Python
|
# -*- coding: utf-8 -*-
|
|||
|
|
"""
|
|||
|
|
文本处理器集合
|
|||
|
|
每个处理器签名：fn(raw: Any, data: IOData) -> Any
|
|||
|
|
可自由组合成处理链，由 IODispatcher 顺序调用
|
|||
|
|
"""
|
|||
|
|
import logging
|
|||
|
|
import re
|
|||
|
|
import time
|
|||
|
|
|
|||
|
|
from uas.io.base_io import IOData
|
|||
|
|
|
|||
|
|
logger = logging.getLogger(__name__)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def processor_clean_text(raw: str, data: IOData) -> str:
|
|||
|
|
"""清洗文本：去除多余空白、特殊字符"""
|
|||
|
|
result = re.sub(r'\s+', ' ', str(raw)).strip()
|
|||
|
|
result = re.sub(r'[^\w\s\u4e00-\u9fff.,!?，。！？、；：""''（）【】]', '', result)
|
|||
|
|
logger.debug(f"[clean] {raw!r} -> {result!r}")
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
def processor_to_upper(raw: str, data: IOData) -> str:
|
|||
|
|
"""英文转大写"""
|
|||
|
|
return raw.upper()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def processor_add_timestamp(raw: str, data: IOData) -> str:
|
|||
|
|
"""追加时间戳"""
|
|||
|
|
ts = time.strftime("%Y-%m-%d %H:%M:%S")
|
|||
|
|
return f"[{ts}] {raw}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def processor_word_count(raw: str, data: IOData) -> str:
|
|||
|
|
"""统计字数并追加到 metadata"""
|
|||
|
|
words = len(raw.split())
|
|||
|
|
chars = len(raw)
|
|||
|
|
data.metadata["word_count"] = words
|
|||
|
|
data.metadata["char_count"] = chars
|
|||
|
|
logger.debug(f"字数统计: words={words} chars={chars}")
|
|||
|
|
return raw # 不修改内容，仅更新 metadata
|
|||
|
|
|
|||
|
|
|
|||
|
|
def processor_keyword_extract(raw: str, data: IOData) -> str:
|
|||
|
|
"""简单关键词提取（示例：提取长度>3的词）"""
|
|||
|
|
words = re.findall(r'\b\w{4,}\b', raw)
|
|||
|
|
keywords = list(set(words))[:10]
|
|||
|
|
data.metadata["keywords"] = keywords
|
|||
|
|
logger.debug(f"关键词: {keywords}")
|
|||
|
|
return raw
|
|||
|
|
|
|||
|
|
|
|||
|
|
def processor_sensitive_filter(raw: str, data: IOData) -> str:
|
|||
|
|
"""敏感词过滤（示例）"""
|
|||
|
|
sensitive = ["badword1", "badword2"]
|
|||
|
|
result = raw
|
|||
|
|
for word in sensitive:
|
|||
|
|
result = result.replace(word, "*" * len(word))
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
def processor_simulate_nlp(raw: str, data: IOData) -> dict:
|
|||
|
|
"""
|
|||
|
|
模拟 NLP 处理（实际可替换为 transformers / spacy 等）
|
|||
|
|
返回结构化结果
|
|||
|
|
"""
|
|||
|
|
import time
|
|||
|
|
time.sleep(0.05) # 模拟耗时
|
|||
|
|
return {
|
|||
|
|
"original": raw,
|
|||
|
|
"length": len(raw),
|
|||
|
|
"summary": raw[:50] + ("..." if len(raw) > 50 else ""),
|
|||
|
|
"intent": "query" if "?" in raw or "？" in raw else "statement",
|
|||
|
|
"lang": "zh" if re.search(r'[\u4e00-\u9fff]', raw) else "en",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ─────────────────────────────────────────────────────────────────
|
|||
|
|
# 处理器工厂：预设处理链
|
|||
|
|
# ─────────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
PIPELINE_PRESETS = {
|
|||
|
|
"basic": [
|
|||
|
|
processor_clean_text,
|
|||
|
|
processor_sensitive_filter,
|
|||
|
|
processor_word_count,
|
|||
|
|
],
|
|||
|
|
"full": [
|
|||
|
|
processor_clean_text,
|
|||
|
|
processor_sensitive_filter,
|
|||
|
|
processor_word_count,
|
|||
|
|
processor_keyword_extract,
|
|||
|
|
processor_simulate_nlp,
|
|||
|
|
],
|
|||
|
|
"speech": [
|
|||
|
|
processor_clean_text,
|
|||
|
|
processor_add_timestamp,
|
|||
|
|
],
|
|||
|
|
}
|