# -*- coding: utf-8 -*- """ 文本处理器集合 每个处理器签名:fn(raw: Any, data: IOData) -> Any 可自由组合成处理链,由 IODispatcher 顺序调用 """ import logging import re import time from uas.io.base_io import IOData logger = logging.getLogger(__name__) def processor_clean_text(raw: str, data: IOData) -> str: """清洗文本:去除多余空白、特殊字符""" result = re.sub(r'\s+', ' ', str(raw)).strip() result = re.sub(r'[^\w\s\u4e00-\u9fff.,!?,。!?、;:""''()【】]', '', result) logger.debug(f"[clean] {raw!r} -> {result!r}") return result def processor_to_upper(raw: str, data: IOData) -> str: """英文转大写""" return raw.upper() def processor_add_timestamp(raw: str, data: IOData) -> str: """追加时间戳""" ts = time.strftime("%Y-%m-%d %H:%M:%S") return f"[{ts}] {raw}" def processor_word_count(raw: str, data: IOData) -> str: """统计字数并追加到 metadata""" words = len(raw.split()) chars = len(raw) data.metadata["word_count"] = words data.metadata["char_count"] = chars logger.debug(f"字数统计: words={words} chars={chars}") return raw # 不修改内容,仅更新 metadata def processor_keyword_extract(raw: str, data: IOData) -> str: """简单关键词提取(示例:提取长度>3的词)""" words = re.findall(r'\b\w{4,}\b', raw) keywords = list(set(words))[:10] data.metadata["keywords"] = keywords logger.debug(f"关键词: {keywords}") return raw def processor_sensitive_filter(raw: str, data: IOData) -> str: """敏感词过滤(示例)""" sensitive = ["badword1", "badword2"] result = raw for word in sensitive: result = result.replace(word, "*" * len(word)) return result def processor_simulate_nlp(raw: str, data: IOData) -> dict: """ 模拟 NLP 处理(实际可替换为 transformers / spacy 等) 返回结构化结果 """ import time time.sleep(0.05) # 模拟耗时 return { "original": raw, "length": len(raw), "summary": raw[:50] + ("..." if len(raw) > 50 else ""), "intent": "query" if "?" in raw or "?" in raw else "statement", "lang": "zh" if re.search(r'[\u4e00-\u9fff]', raw) else "en", } # ───────────────────────────────────────────────────────────────── # 处理器工厂:预设处理链 # ───────────────────────────────────────────────────────────────── PIPELINE_PRESETS = { "basic": [ processor_clean_text, processor_sensitive_filter, processor_word_count, ], "full": [ processor_clean_text, processor_sensitive_filter, processor_word_count, processor_keyword_extract, processor_simulate_nlp, ], "speech": [ processor_clean_text, processor_add_timestamp, ], }