feat: implement LLM keyword extraction and fusion scoring for table matching

- Add extract_keywords() to LLMService with conversation context support,
  using process_stream + start_log/end_log for logging consistency
- Add expand_with_terminology() to expand keywords with synonym matching
- Add fusion scoring in calc_table_embedding(): α * vec_score + (1-α) * keyword_score
- Add calc_keyword_score() with multi-level matching:
  table name exact (1.0), comment exact (0.9), comment substring (0.7),
  table name partial (0.25-0.5), field comment exact (0.5),
  field comment substring (0.4), field name partial (0.3)
- Scoring considers keyword coverage rate to differentiate tables
  with different numbers of matched keywords
- Extracted keywords persisted to ChatRecord for reuse across endpoints
- get_terminology_template() supports comma-separated keyword input
  with per-keyword matching for higher precision
- Add TABLE_EMBEDDING_ALPHA and TABLE_EMBEDDING_KEYWORD_ENABLED config
This commit is contained in:
ulleo
2026-09-14 17:51:45 +08:00
committed by ulleo
parent aac200aef5
commit 3fd6fa867d
15 changed files with 1200 additions and 29 deletions
+4 -4
View File
@@ -24,14 +24,14 @@ config = context.config
# from apps.system.models.user import SQLModel # noqa
# from apps.settings.models.setting_models import SQLModel
#from apps.chat.models.chat_model import SQLModel
from apps.terminology.models.terminology_model import SQLModel
from sqlbot_xpack.custom_prompt.models.custom_prompt_model import SQLModel
from apps.chat.models.chat_model import SQLModel
#from apps.terminology.models.terminology_model import SQLModel
#from sqlbot_xpack.custom_prompt.models.custom_prompt_model import SQLModel
#from apps.data_training.models.data_training_model import SQLModel
# from apps.dashboard.models.dashboard_model import SQLModel
from common.core.config import settings # noqa
#from apps.datasource.models.datasource import SQLModel
from apps.system.models.system_model import SQLModel
#from apps.system.models.system_model import SQLModel
target_metadata = SQLModel.metadata
@@ -0,0 +1,31 @@
"""072_chat_record
Revision ID: f898322341a5
Revises: a2e2ecfa5a9c
Create Date: 2026-09-14 15:54:47.921806
"""
from alembic import op
import sqlalchemy as sa
import sqlmodel.sql.sqltypes
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = 'f898322341a5'
down_revision = 'a2e2ecfa5a9c'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('chat_record', sa.Column('extracted_keywords', sa.Text(), nullable=True, comment='关键词'))
op.add_column('chat_record', sa.Column('expanded_keywords', sa.Text(), nullable=True, comment='扩展后的关键词'))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('chat_record', 'extracted_keywords')
op.drop_column('chat_record', 'expanded_keywords')
# ### end Alembic commands ###
+13
View File
@@ -933,6 +933,19 @@ def save_sql_answer(session: SessionDep, record_id: int, answer: str) -> ChatRec
return record
def save_extracted_keywords(session: SessionDep, record_id: int, expanded_keywords: str,
extracted_keywords: str = None) -> None:
"""保存提取的关键词到 record,供后续独立接口复用。"""
if not record_id:
return
values = {'expanded_keywords': expanded_keywords}
if extracted_keywords:
values['extracted_keywords'] = extracted_keywords
stmt = update(ChatRecord).where(and_(ChatRecord.id == record_id)).values(**values)
session.execute(stmt)
session.commit()
def save_analysis_answer(session: SessionDep, record_id: int, answer: str = '') -> ChatRecord:
if not record_id:
raise Exception("Record id cannot be None")
+9
View File
@@ -47,6 +47,7 @@ class OperationEnum(Enum):
FILTER_CUSTOM_PROMPT = '11'
EXECUTE_SQL = '12'
GENERATE_PICTURE = '13'
EXTRACT_KEYWORDS = '14'
class ChatFinishStep(Enum):
@@ -129,6 +130,8 @@ class ChatRecord(SQLModel, table=True):
analysis_record_id: int = Field(sa_column=Column(BigInteger, nullable=True))
predict_record_id: int = Field(sa_column=Column(BigInteger, nullable=True))
regenerate_record_id: int = Field(sa_column=Column(BigInteger, nullable=True))
extracted_keywords: str = Field(sa_column=Column(Text, nullable=True))
expanded_keywords: str = Field(sa_column=Column(Text, nullable=True))
class ChatRecordResult(BaseModel):
@@ -238,6 +241,8 @@ class AiModelQuestion(BaseModel):
question: str = None
ai_modal_id: int = None
ai_modal_name: str = None # Specific model name
extracted_keywords: str = "" # LLM 提取的核心业务关键词(逗号分隔,未经扩展)
expanded_keywords: str = "" # 经术语同义词扩展后的关键词(用于表匹配)
engine: str = ""
db_schema: str = ""
sql: str = ""
@@ -255,6 +260,10 @@ class AiModelQuestion(BaseModel):
sample_data: str = ""
sqlbot_name: str = "SQLBot"
def extract_keywords_sys_prompt(self) -> str:
return get_sql_template().get("extract_keywords").format(lang=self.lang,
sqlbot_name=self.sqlbot_name)
def sql_sys_question(self, db_type: Union[str, DB], enable_query_limit: bool = True):
templates: dict[str, str] = {}
_sql_template = get_sql_example_template(db_type)
+137 -4
View File
@@ -2,6 +2,7 @@ import asyncio
import concurrent
import json
import os
import re
import traceback
import urllib.parse
import warnings
@@ -30,7 +31,7 @@ from apps.ai_model.model_factory import LLMConfig, LLMFactory, get_default_confi
from apps.chat.curd.chat import save_question, save_sql_answer, save_sql, \
save_error_message, save_sql_exec_data, save_chart_answer, save_chart, \
finish_record, save_analysis_answer, save_predict_answer, save_predict_data, \
save_select_datasource_answer, save_recommend_question_answer, \
save_select_datasource_answer, save_recommend_question_answer, save_extracted_keywords, \
get_old_questions, save_analysis_predict_record, rename_chat, get_chart_config, \
get_chat_chart_data, list_generate_sql_logs, list_generate_chart_logs, start_log, end_log, \
get_last_execute_sql_error, format_json_data, format_chart_fields, get_chat_brief_generate, get_chat_predict_data, \
@@ -48,6 +49,7 @@ from apps.system.crud.assistant import AssistantOutDs, AssistantOutDsFactory, ge
from apps.system.crud.parameter_manage import get_groups
from apps.system.crud.user import user_ws_list
from apps.system.schemas.system_schema import AssistantOutDsSchema
from apps.terminology.curd.terminology import expand_with_terminology
from apps.terminology.curd.terminology import get_terminology_template
from common.core.config import settings
from common.core.db import engine
@@ -378,14 +380,16 @@ class LLMService:
calculate_oid = self.current_assistant.oid if self.current_assistant.type != 4 else self.oid
if self.current_assistant.type == 1:
calculate_ds_id = None
# 使用原始提取的关键词(未扩展)进行术语匹配,避免同义词重复查询
match_text = self.chat_question.extracted_keywords or self.chat_question.question
if self.current_assistant and self.current_assistant.type == 1:
self.chat_question.terminologies, term_list = get_terminology_template(_session,
self.chat_question.question,
match_text,
calculate_oid,
None, self.current_assistant.id)
else:
self.chat_question.terminologies, term_list = get_terminology_template(_session,
self.chat_question.question,
match_text,
calculate_oid,
calculate_ds_id)
@@ -447,17 +451,108 @@ class LLMService:
OperationEnum.FILTER_SQL_EXAMPLE],
full_message=example_list)
def extract_keywords(self, _session: Session) -> tuple[bool, str]:
"""从用户问题中提取核心业务实体词。
使用 LLM 提取名词性业务实体,忽略时间词、动作词、修饰词。
支持上下文历史,可根据对话上下文理解用户意图。
提取结果可复用于表匹配、术语扩展等环节。
Returns:
(is_error, result) 元组:
- (False, keywords) 成功,关键词逗号分隔
- (True, error_message) 错误(LLM 失败或非查数据意图)
"""
system_prompt = self.chat_question.extract_keywords_sys_prompt()
keywords_msg: List[Union[BaseMessage, dict[str, Any]]] = []
keywords_msg.append(SystemPromptMessage(content=system_prompt))
# 加载上下文历史(仅取用户的历史提问,不含 AI 生成的 SQL 等回复)
last_sql_messages: List[dict[str, Any]] = self.generate_sql_logs[-1].messages if len(
self.generate_sql_logs) > 0 else []
if self.chat_question.regenerate_record_id:
_temp_log = next(
filter(lambda obj: obj.pid == self.chat_question.regenerate_record_id, self.generate_sql_logs), None)
last_sql_messages: List[dict[str, Any]] = _temp_log.messages if _temp_log else []
# 排除系统提示词和 AI 回复,只保留用户的历史提问
last_user_messages = [
obj for obj in last_sql_messages
if obj.get("sqlbot_system") != True and obj.get('type') == 'human'
]
if last_user_messages:
last_rounds = get_last_conversation_rounds(last_user_messages, rounds=self.base_message_round_count_limit)
for _msg_dict in last_rounds:
content = _msg_dict.get('content', '')
# 提取 <user-question> 标签内的用户原始提问,排除 error-msg 等干扰信息
match = re.search(r'<user-question>(.*?)</user-question>', content, re.DOTALL)
if match:
question_text = match.group(1).strip()
if question_text:
keywords_msg.append(HumanMessage(content=question_text))
# 当前问题
keywords_msg.append(HumanMessage(content=self.chat_question.question))
self.current_logs[OperationEnum.EXTRACT_KEYWORDS] = start_log(
session=_session,
ai_modal_id=self.chat_question.ai_modal_id,
ai_modal_name=self.chat_question.ai_modal_name,
operate=OperationEnum.EXTRACT_KEYWORDS,
record_id=self.record.id,
full_message=[{'type': msg.type, 'content': msg.content} for msg in keywords_msg]
)
full_thinking_text = ''
full_text = ''
token_usage = {}
res = process_stream(self.llm.stream(keywords_msg), token_usage)
for chunk in res:
if chunk.get('content'):
full_text += chunk.get('content')
if chunk.get('reasoning_content'):
full_thinking_text += chunk.get('reasoning_content')
result_text = full_text.strip()
keywords_msg.append(AIMessage(result_text))
self.current_logs[OperationEnum.EXTRACT_KEYWORDS] = end_log(
session=_session,
log=self.current_logs[OperationEnum.EXTRACT_KEYWORDS],
full_message=[{'type': msg.type, 'content': msg.content} for msg in keywords_msg],
reasoning_content=full_thinking_text if full_thinking_text else None,
token_usage=token_usage
)
if result_text.startswith('ERROR:NOT_DATA_QUERY'):
# return True, '您的问题似乎不是查询数据的问题,请提出与数据查询相关的问题。'
return False, ''
if result_text == 'EMPTY' or not result_text:
self.chat_question.extracted_keywords = self.chat_question.question
return False, ''
self.chat_question.extracted_keywords = result_text
return False, result_text
def choose_table_schema(self, _session: Session):
self.current_logs[OperationEnum.CHOOSE_TABLE] = start_log(session=_session,
operate=OperationEnum.CHOOSE_TABLE,
record_id=self.record.id,
local_operation=True)
# 使用扩展后的关键词进行表匹配(包含同义词,匹配更全面)
keywords = self.chat_question.expanded_keywords or self.chat_question.question
self.chat_question.db_schema, tables = self.out_ds_instance.get_db_schema(
self.ds.id, self.chat_question.question) if self.out_ds_instance else get_table_schema(
session=_session,
current_user=self.current_user,
ds=self.ds,
question=self.chat_question.question)
question=self.chat_question.question,
keywords=keywords)
# Get sample data for all tables
if not self.out_ds_instance:
@@ -479,6 +574,12 @@ class LLMService:
self.chat_question.data = orjson.dumps(data.get('data')).decode()
analysis_msg: List[Union[BaseMessage, dict[str, Any]]] = []
# 从 record 加载已提取的关键词(SQL 生成阶段已持久化)
if self.record.extracted_keywords:
self.chat_question.extracted_keywords = self.record.extracted_keywords
if self.record.expanded_keywords:
self.chat_question.expanded_keywords = self.record.expanded_keywords
ds_id = self.ds.id if isinstance(self.ds, CoreDatasource) else None
self.filter_terminology_template(_session, self.oid, ds_id)
@@ -781,6 +882,22 @@ class LLMService:
oid = self.ds.oid if isinstance(self.ds, CoreDatasource) else 1
ds_id = self.ds.id if isinstance(self.ds, CoreDatasource) else None
# 提取关键词(前置步骤,结果复用于术语匹配和表匹配)
if settings.TABLE_EMBEDDING_KEYWORD_ENABLED:
is_error, kw_result = self.extract_keywords(_session)
if is_error:
raise SingleMessageError(kw_result)
if kw_result:
self.chat_question.extracted_keywords = kw_result
SQLBotLogUtil.info(f"提取的关键词: {kw_result}")
# 术语同义词扩展(用于表匹配,不用于术语模板查询)
expanded = expand_with_terminology(kw_result, _session, oid)
self.chat_question.expanded_keywords = expanded
SQLBotLogUtil.info(f"扩展后的关键词: {expanded}")
# 持久化到 record,供后续独立接口(如 generate_analysis)使用
save_extracted_keywords(_session, self.record.id,
expanded_keywords=expanded, extracted_keywords=kw_result)
self.filter_terminology_template(_session, oid, ds_id)
self.filter_training_template(_session, oid, ds_id)
@@ -1247,6 +1364,22 @@ class LLMService:
oid = self.ds.oid if isinstance(self.ds, CoreDatasource) else 1
ds_id = self.ds.id if isinstance(self.ds, CoreDatasource) else None
# 提取关键词(前置步骤,结果复用于术语匹配和表匹配)
if settings.TABLE_EMBEDDING_KEYWORD_ENABLED:
is_error, kw_result = self.extract_keywords(_session)
if is_error:
raise SingleMessageError(kw_result)
if kw_result:
self.chat_question.extracted_keywords = kw_result
SQLBotLogUtil.info(f"提取的关键词: {kw_result}")
# 术语同义词扩展(用于表匹配,不用于术语模板查询)
expanded = expand_with_terminology(kw_result, _session, oid)
self.chat_question.expanded_keywords = expanded
SQLBotLogUtil.info(f"扩展后的关键词: {expanded}")
# 持久化到 record,供后续独立接口(如 generate_analysis)使用
save_extracted_keywords(_session, self.record.id,
expanded_keywords=expanded, extracted_keywords=kw_result)
self.filter_terminology_template(_session, oid, ds_id)
self.filter_training_template(_session, oid, ds_id)
+18 -4
View File
@@ -517,7 +517,8 @@ def get_tables_sample_data(session: SessionDep, current_user: CurrentUser, ds: C
def get_table_schema(session: SessionDep, current_user: CurrentUser, ds: CoreDatasource, question: str,
embedding: bool = True, table_list: list[str] = None) -> tuple[str, list]:
embedding: bool = True, table_list: list[str] = None,
keywords: str = None) -> tuple[str, list]:
schema_str = ""
table_objs = get_table_obj_by_ds(session=session, current_user=current_user, ds=ds)
if len(table_objs) == 0:
@@ -538,17 +539,25 @@ def get_table_schema(session: SessionDep, current_user: CurrentUser, ds: CoreDat
table_comment = ''
if obj.table.custom_comment:
table_comment = obj.table.custom_comment.strip()
if not table_comment and obj.table.table_comment:
table_comment = obj.table.table_comment.strip()
if table_comment == '':
schema_table += '\n[\n'
else:
schema_table += f", {table_comment}\n[\n"
fields_info = []
if obj.fields:
field_list = []
for field in obj.fields:
field_comment = ''
if field.custom_comment:
field_comment = field.custom_comment.strip()
fields_info.append({
"field_name": field.field_name,
"field_comment": (field.field_comment or '').strip(),
"custom_comment": (field.custom_comment or '').strip()
})
if field_comment == '':
field_list.append(f"({field.field_name}:{field.field_type})")
else:
@@ -557,7 +566,7 @@ def get_table_schema(session: SessionDep, current_user: CurrentUser, ds: CoreDat
schema_table += '\n]\n'
t_obj = {"id": obj.table.id, "table_name": obj.table.table_name, "schema_table": schema_table,
"embedding": obj.table.embedding}
"embedding": obj.table.embedding, "table_comment": table_comment, "fields": fields_info}
tables.append(t_obj)
all_tables.append(t_obj)
@@ -565,9 +574,14 @@ def get_table_schema(session: SessionDep, current_user: CurrentUser, ds: CoreDat
if not tables:
return schema_str, []
# do table embedding
# 执行表 embedding 匹配
if embedding and tables and settings.TABLE_EMBEDDING_ENABLED:
tables = calc_table_embedding(tables, question)
try:
tables = calc_table_embedding(tables, question, session=session, oid=current_user.oid,
keywords=keywords)
except ValueError as e:
# LLM 判断意图不是查数据或关键词提取失败
return str(e), []
# splice schema
if tables:
for s in tables:
@@ -1,6 +1,7 @@
# Author: Junjun
# Date: 2025/9/23
import json
import re
import time
import traceback
@@ -10,6 +11,15 @@ from common.core.config import settings
from common.utils.utils import SQLBotLogUtil
def _parse_embedding(embedding):
"""解析 embedding,兼容 JSON 字符串和原生列表两种格式。"""
if isinstance(embedding, list):
return embedding
if isinstance(embedding, str):
return json.loads(embedding)
return None
def get_table_embedding(tables: list[dict], question: str):
_list = []
for table in tables:
@@ -40,38 +50,244 @@ def get_table_embedding(tables: list[dict], question: str):
return _list
def calc_table_embedding(tables: list[dict], question: str):
def calc_keyword_score(
question: str,
table_name: str,
table_comment: str,
fields: list[dict]
) -> float:
"""计算关键词与表的匹配分数。
综合考虑所有关键词的匹配情况:
- 表名完整匹配:1.0
- 表注释完整匹配:0.9
- 部分匹配时,按匹配质量 * 关键词覆盖率计算
Args:
question: 提取后的关键词(逗号分隔)或原始问题
table_name: 表名
table_comment: 表注释(已合并 custom_comment 优先)
fields: 字段列表,每个 dict 包含 field_name, field_comment, custom_comment
Returns:
0.0~1.0 的匹配分数
"""
if not question:
return 0.0
keywords = [kw.strip().lower() for kw in question.split(',') if kw.strip()]
if not keywords:
return 0.0
tn = table_name.lower()
tc = (table_comment or '').lower()
total_kw = len(keywords)
# 收集每个关键词的最佳匹配分数
kw_scores = {} # keyword -> best_score
for kw in keywords:
# 1. 表名完整匹配:1.0
if kw == tn:
kw_scores[kw] = 1.0
continue
# 2. 表注释完整匹配:0.9
if tc and kw == tc:
kw_scores[kw] = max(kw_scores.get(kw, 0), 0.9)
continue
# 3. 表注释子串匹配:0.7(关键词是注释的子串,如 "用户" 匹配 "用户表")
if tc and kw in tc:
kw_scores[kw] = max(kw_scores.get(kw, 0), 0.7)
continue
# 4. 表名部分匹配:收集匹配的关键词
tn_parts = set(re.split(r'[_\s]+', tn))
tn_parts.discard('')
if kw in tn_parts:
kw_scores[kw] = max(kw_scores.get(kw, 0), 0) # 标记匹配,分数稍后计算
# 4. 字段注释匹配:0.5(精确)/ 0.4(子串)
for field in fields:
field_comment = ''
if field.get('custom_comment'):
field_comment = field['custom_comment'].strip().lower()
if not field_comment and field.get('field_comment'):
field_comment = field['field_comment'].strip().lower()
if field_comment:
if kw == field_comment:
kw_scores[kw] = max(kw_scores.get(kw, 0), 0.5)
break
elif kw in field_comment:
kw_scores[kw] = max(kw_scores.get(kw, 0), 0.4)
# 5. 字段名匹配:0.3
if kw not in kw_scores or kw_scores[kw] < 0.3:
for field in fields:
fname = field.get('field_name', '').lower()
if fname:
fname_parts = set(re.split(r'[_\s]+', fname))
fname_parts.discard('')
if kw in fname_parts:
kw_scores[kw] = max(kw_scores.get(kw, 0), 0.3)
break
if not kw_scores:
return 0.0
# 表名/注释完整匹配 → 直接返回 1.0 或 0.9
if kw_scores.get(tn) == 1.0:
return 1.0
if tc and kw_scores.get(tc) == 0.9:
return 0.9
# 表名部分匹配:按覆盖率计算(多个关键词共同覆盖表名)
tn_parts = set(re.split(r'[_\s]+', tn))
tn_parts.discard('')
if tn_parts:
matched_parts = {kw for kw in keywords if kw in tn_parts}
if matched_parts:
coverage = len(matched_parts) / len(tn_parts)
tn_score = round(0.25 + 0.25 * coverage, 2)
# 更新这些关键词的分数为表名匹配分数
for kw in matched_parts:
kw_scores[kw] = max(kw_scores.get(kw, 0), tn_score)
# 综合评分:最佳匹配分数 * 关键词覆盖率
best_score = max(kw_scores.values())
match_rate = len(kw_scores) / total_kw
return round(best_score * match_rate, 2)
def calc_table_embedding(tables: list[dict], question: str, session=None, oid: int = None,
keywords: str = None):
"""使用向量相似度和关键词匹配的融合评分计算表的相关性分数。
当 keywords 不为空时(已由调用方提取并扩展):
1. 用 keywords 计算向量相似度(vec_score)
2. 计算关键词匹配度(keyword_score)
3. 融合评分:final_score = α * vec_score + (1-α) * keyword_score
4. keyword_score=1.0 的表保证排在最前面
当 keywords 为空或 TABLE_EMBEDDING_KEYWORD_ENABLED 为 False 时,回退到纯向量匹配。
Args:
tables: 表列表,每个 dict 包含 id, table_name, schema_table, embedding, table_comment, fields
question: 原始用户问题(用于纯向量匹配回退)
session: 数据库会话(预留)
oid: 组织 ID(预留)
keywords: 已提取并扩展的关键词(由调用方传入)
"""
if not keywords or not settings.TABLE_EMBEDDING_KEYWORD_ENABLED:
# 回退到纯向量匹配
return _calc_vector_only(tables, question)
_list = []
for table in tables:
_list.append(
{"id": table.get('id'), "schema_table": table.get('schema_table'), "embedding": table.get('embedding'),
"cosine_similarity": 0.0, "table_name": table.get('table_name')})
_list.append({
"id": table.get('id'),
"schema_table": table.get('schema_table'),
"embedding": table.get('embedding'),
"cosine_similarity": 0.0,
"table_name": table.get('table_name'),
"table_comment": table.get('table_comment', ''),
"fields": table.get('fields', [])
})
if not _list:
return _list
try:
start_time = time.time()
# 步骤 1:用关键词计算向量相似度
model = EmbeddingModelCache.get_model()
results = [item.get('embedding') for item in _list]
q_embedding = model.embed_query(keywords)
for index in range(len(results)):
item = results[index]
if item:
_list[index]['cosine_similarity'] = cosine_similarity(q_embedding, _parse_embedding(item))
# 步骤 2 & 3:计算关键词分数并融合
alpha = settings.TABLE_EMBEDDING_ALPHA
for table in _list:
vec_score = table['cosine_similarity']
kw_score = calc_keyword_score(
keywords,
table.get('table_name', ''),
table.get('table_comment', ''),
table.get('fields', [])
)
table['keyword_score'] = kw_score
table['cosine_similarity'] = alpha * vec_score + (1 - alpha) * kw_score
# 步骤 4:排序 - 精确匹配(keyword_score=1.0)始终排最前
exact_matches = [t for t in _list if t.get('keyword_score') == 1.0]
other_tables = [t for t in _list if t.get('keyword_score') != 1.0]
exact_matches.sort(key=lambda x: x['cosine_similarity'], reverse=True)
other_tables.sort(key=lambda x: x['cosine_similarity'], reverse=True)
_list = exact_matches + other_tables
_list = _list[:settings.TABLE_EMBEDDING_COUNT]
end_time = time.time()
SQLBotLogUtil.info(f"融合评分耗时 {end_time - start_time:.3f}s")
SQLBotLogUtil.info(json.dumps([{
"id": ele.get('id'),
"schema_table": ele.get('schema_table'),
"cosine_similarity": ele.get('cosine_similarity'),
"keyword_score": ele.get('keyword_score'),
"table_name": ele.get('table_name')
} for ele in _list]))
return _list
except Exception:
traceback.print_exc()
# 异常回退到纯向量匹配
return _calc_vector_only(tables, question)
def _calc_vector_only(tables: list[dict], question: str):
"""纯向量相似度评分(原始逻辑)。"""
_list = []
for table in tables:
_list.append({
"id": table.get('id'),
"schema_table": table.get('schema_table'),
"embedding": table.get('embedding'),
"cosine_similarity": 0.0,
"table_name": table.get('table_name'),
"table_comment": table.get('table_comment', ''),
"fields": table.get('fields', [])
})
if _list:
try:
# text = [s.get('schema_table') for s in _list]
#
model = EmbeddingModelCache.get_model()
start_time = time.time()
# results = model.embed_documents(text)
# end_time = time.time()
# SQLBotLogUtil.info(str(end_time - start_time))
results = [item.get('embedding') for item in _list]
q_embedding = model.embed_query(question)
for index in range(len(results)):
item = results[index]
if item:
_list[index]['cosine_similarity'] = cosine_similarity(q_embedding, json.loads(item))
_list[index]['cosine_similarity'] = cosine_similarity(q_embedding, _parse_embedding(item))
_list.sort(key=lambda x: x['cosine_similarity'], reverse=True)
_list = _list[:settings.TABLE_EMBEDDING_COUNT]
# print(len(_list))
end_time = time.time()
SQLBotLogUtil.info(str(end_time - start_time))
SQLBotLogUtil.info(json.dumps([{"id": ele.get('id'), "schema_table": ele.get('schema_table'),
"cosine_similarity": ele.get('cosine_similarity'), "table_name": ele.get('table_name')}
for ele in _list]))
SQLBotLogUtil.info(json.dumps([{
"id": ele.get('id'),
"schema_table": ele.get('schema_table'),
"cosine_similarity": ele.get('cosine_similarity'),
"table_name": ele.get('table_name')
} for ele in _list]))
return _list
except Exception:
traceback.print_exc()
+114 -2
View File
@@ -977,10 +977,122 @@ def get_terminology_template(session: SessionDep, question: str, oid: Optional[i
advanced_application_id: Optional[int] = None) -> tuple[str, list[dict]]:
if not oid:
oid = 1
_results = select_terminology_by_word(session, question, oid, datasource, advanced_application_id)
if _results and len(_results) > 0:
# 有逗号时视为关键词列表,逐个匹配(精度更高,避免整体 ILIKE 误匹配)
# 无逗号时视为自然语言问题,整体匹配
if ',' in question:
parts = [kw.strip() for kw in question.split(',') if kw.strip()]
else:
parts = [question]
# select_terminology_by_word 返回 list[dict],每个 dict 结构:
# {'words': ['销售', 'sales'], 'description': '...'}
seen_words = set()
_results = []
for part in parts:
for item in select_terminology_by_word(session, part, oid, datasource, advanced_application_id):
item_words = set(item.get('words', [])) if isinstance(item, dict) else set()
if item_words and not item_words.issubset(seen_words):
seen_words.update(item_words)
_results.append(item)
if _results:
terminology = to_xml_string(_results)
template = get_base_terminology_template().format(terminologies=terminology)
return template, _results
else:
return '', []
def expand_with_terminology(keywords: str, session: SessionDep, oid: int) -> str:
"""使用术语同义词扩展关键词。
查询当前租户启用的术语表,将匹配的关键词扩展为同义词集合。
pid=None 表示父术语(主词),pid=<id> 表示子术语(同义词)。
Args:
keywords: 逗号分隔的关键词字符串
session: 数据库会话
oid: 组织 ID
Returns:
扩展后的关键词字符串(逗号分隔),最多 10 个关键词
"""
if not keywords or not session or oid is None:
return keywords
keyword_list = [kw.strip() for kw in keywords.split(',') if kw.strip()]
if not keyword_list:
return keywords
try:
# 查询当前租户启用的术语(限制 500 条避免内存问题)
terms = session.query(Terminology).filter(
Terminology.oid == oid,
Terminology.enabled == True
).limit(500).all()
if not terms:
return keywords
# 构建父子结构
parent_map = {} # id -> Terminology(仅父术语)
children_map = {} # parent_id -> [Terminology](子术语列表)
word_to_parent_ids = {} # 小写词 -> 父术语 id 集合
for term in terms:
if term.pid is None or term.pid == 0:
parent_map[term.id] = term
if term.word:
w = term.word.strip().lower()
if w not in word_to_parent_ids:
word_to_parent_ids[w] = set()
word_to_parent_ids[w].add(term.id)
else:
if term.pid not in children_map:
children_map[term.pid] = []
children_map[term.pid].append(term)
if term.word:
w = term.word.strip().lower()
if w not in word_to_parent_ids:
word_to_parent_ids[w] = set()
word_to_parent_ids[w].add(term.pid)
# 扩展关键词
expanded = list(keyword_list)
seen = set(kw.lower() for kw in keyword_list)
max_keywords = 10
for kw in keyword_list:
if len(expanded) >= max_keywords:
break
kw_lower = kw.lower()
matched_parent_ids = word_to_parent_ids.get(kw_lower, set())
for pid in matched_parent_ids:
if len(expanded) >= max_keywords:
break
# 添加父术语
parent = parent_map.get(pid)
if parent and parent.word:
parent_word = parent.word.strip()
if parent_word.lower() not in seen:
expanded.append(parent_word)
seen.add(parent_word.lower())
# 添加子术语(同义词)
for child in children_map.get(pid, []):
if len(expanded) >= max_keywords:
break
if child.word:
child_word = child.word.strip()
if child_word.lower() not in seen:
expanded.append(child_word)
seen.add(child_word.lower())
return ','.join(expanded)
except Exception:
traceback.print_exc()
return keywords
+13
View File
@@ -128,6 +128,8 @@ class Settings(BaseSettings):
TABLE_EMBEDDING_ENABLED: bool = True
TABLE_EMBEDDING_COUNT: int = 10
TABLE_EMBEDDING_ALPHA: float = 0.4 # weight for vector score; (1-alpha) is keyword weight
TABLE_EMBEDDING_KEYWORD_ENABLED: bool = True
DS_EMBEDDING_COUNT: int = 10
ORACLE_CLIENT_PATH: str = '/opt/sqlbot/db_client/oracle_instant_client'
@@ -138,6 +140,7 @@ class Settings(BaseSettings):
'PARSE_REASONING_BLOCK_ENABLED',
'PG_POOL_PRE_PING',
'TABLE_EMBEDDING_ENABLED',
'TABLE_EMBEDDING_KEYWORD_ENABLED',
mode='before')
@classmethod
def lowercase_bool(cls, v: Any) -> Any:
@@ -150,5 +153,15 @@ class Settings(BaseSettings):
return False
return v
@field_validator('TABLE_EMBEDDING_ALPHA', mode='before')
@classmethod
def clamp_alpha(cls, v: Any) -> float:
"""将 TABLE_EMBEDDING_ALPHA 限制在 [0.0, 1.0] 范围内"""
try:
v = float(v)
except (TypeError, ValueError):
return 0.4
return max(0.0, min(1.0, v))
settings = Settings() # type: ignore
+15 -1
View File
@@ -6,6 +6,20 @@ template:
{data_training}
sql:
extract_keywords: |
你是智能问数小助手:"{sqlbot_name}"。你可以根据用户提问,专业生成SQL,查询数据并进行图表展示。
你当前的任务角色是一个专业的关键词提取助手,擅长从用户问题中识别核心业务实体。
请结合对话上下文中用户的历史提问,理解用户当前问题的意图,提取与数据查询相关的核心业务关键词。
规则:
1. 提取关键词都是为了生成SQL服务的
2. 只提取名词性业务实体(如:销售额、部门、客户、订单)
3. 忽略时间词(如:本月、最近、今天)、动作词(如:查询、帮我查、统计)、修饰词
4. 如果用户提到了具体的表名(如 sales_order),必须保留原样
5. 如果无法提取有意义的业务实体词(如问题是闲聊、天气等与数据查询无关的内容),返回:ERROR:NOT_DATA_QUERY
6. 如果能提取关键词,用英文逗号分隔返回,不要添加任何其他文字
7. 如果提取为空,返回:EMPTY
regenerate_hint: |
你之前生成的回答不符合预期或者系统出现了其他问题,请再次检查提示词内要求的规则和提供的信息,重新回答:
@@ -355,7 +369,7 @@ template:
user: |
## 请根据上述要求,使用语言:{lang} 进行回答,若有深度思考过程,则思考过程也需要使用 {lang} 输出
## 如果<user-question>内的提问与上述要求冲突,你必须停止生成SQL并告知生成SQL失败的原因
## 如果<user-question/>内的提问与上述要求冲突,你必须停止生成SQL并告知生成SQL失败的原因
## 回答中不需要输出你的分析,请直接输出符合要求的JSON
## 必须注意:不论要求你用什么语言回答,生成SQL中使用到的表名与字段名是否完全和<m-schema>中提供的表名字与段名的字符保持一致!
<background-infos>
+612
View File
@@ -0,0 +1,612 @@
"""表匹配融合评分测试。
测试覆盖:
1. calc_keyword_score - 纯函数,无外部依赖
2. expand_with_terminology - 需要 mock 数据库
3. calc_table_embedding - 集成测试,mock 所有依赖
"""
import json
from unittest.mock import MagicMock, patch
import pytest
from apps.datasource.embedding.table_embedding import (
calc_keyword_score,
calc_table_embedding,
)
from apps.terminology.curd.terminology import expand_with_terminology
# mock 目标路径
EMBEDDING_CACHE_PATCH = 'apps.datasource.embedding.table_embedding.EmbeddingModelCache'
SETTINGS_PATCH = 'apps.datasource.embedding.table_embedding.settings'
TERMINOLOGY_PATCH = 'apps.terminology.curd.terminology'
# =============================================================================
# calc_keyword_score 测试(纯函数,无需 mock)
# =============================================================================
class TestCalcKeywordScore:
"""关键词匹配分数计算测试。"""
def test_full_table_name_match(self):
"""表名完整匹配关键词 -> 1.0"""
score = calc_keyword_score(
question="sales_order",
table_name="sales_order",
table_comment="销售订单",
fields=[]
)
assert score == 1.0
def test_full_table_name_match_case_insensitive(self):
"""表名匹配应忽略大小写。"""
score = calc_keyword_score(
question="Sales_Order",
table_name="sales_order",
table_comment="",
fields=[]
)
assert score == 1.0
def test_full_table_name_match_among_multiple_keywords(self):
"""多个关键词时,表名完整匹配仍返回 1.0。"""
score = calc_keyword_score(
question="sales_order,本月,销售额",
table_name="sales_order",
table_comment="",
fields=[]
)
assert score == 1.0
def test_table_comment_full_match(self):
"""表注释完整匹配关键词 -> 0.9"""
score = calc_keyword_score(
question="销售订单",
table_name="so_header",
table_comment="销售订单",
fields=[]
)
assert score == 0.9
def test_table_comment_full_match_custom_comment_priority(self):
"""自定义备注应作为 table_comment 使用(调用方负责合并)。"""
# 调用方传入已合并的注释,这里模拟该场景
score = calc_keyword_score(
question="销售订单",
table_name="so_header",
table_comment="自定义备注",
fields=[]
)
# "销售订单" != "自定义备注",不匹配注释
assert score == 0.0
def test_partial_table_name_match_single_keyword(self):
"""关键词匹配表名部分 -> 0.25~0.5(按覆盖比例)。"""
# "sales" 匹配 "sales_order" 的一部分(2 个部分:sales, order)
# 覆盖率 = 1/2 = 0.5 -> 0.25 + 0.25*0.5 = 0.375
score = calc_keyword_score(
question="sales",
table_name="sales_order",
table_comment="",
fields=[]
)
assert score == 0.38 # 0.25 + 0.25 * 0.5 = 0.375,四舍五入为 0.38
def test_partial_table_name_match_full_coverage(self):
"""表名所有部分都被关键词覆盖 -> 0.5。"""
# "sales" 和 "order" 覆盖了 "sales_order" 的所有部分
score = calc_keyword_score(
question="sales,order",
table_name="sales_order",
table_comment="",
fields=[]
)
assert score == 0.5
def test_field_comment_match(self):
"""关键词匹配字段注释 -> 0.5"""
fields = [
{"field_name": "amount", "field_comment": "销售额", "custom_comment": ""},
{"field_name": "qty", "field_comment": "数量", "custom_comment": ""},
]
score = calc_keyword_score(
question="销售额",
table_name="order_detail",
table_comment="",
fields=fields
)
assert score == 0.5
def test_field_custom_comment_priority(self):
"""自定义注释应优先于字段注释。"""
fields = [
{"field_name": "amount", "field_comment": "原始注释", "custom_comment": "销售额"},
]
score = calc_keyword_score(
question="销售额",
table_name="order_detail",
table_comment="",
fields=fields
)
assert score == 0.5
def test_field_name_match(self):
"""关键词匹配字段名部分 -> 0.3"""
fields = [
{"field_name": "order_amount", "field_comment": "", "custom_comment": ""},
]
score = calc_keyword_score(
question="amount",
table_name="transactions",
table_comment="",
fields=fields
)
assert score == 0.3
def test_field_name_match_no_duplicate(self):
"""同一词根匹配多个字段时只计算一次。"""
fields = [
{"field_name": "order_amount", "field_comment": "", "custom_comment": ""},
{"field_name": "total_amount", "field_comment": "", "custom_comment": ""},
]
# "amount" 匹配了两个字段,但仍应返回 0.3(不会更高)
score = calc_keyword_score(
question="amount",
table_name="transactions",
table_comment="",
fields=fields
)
assert score == 0.3
def test_no_match(self):
"""无匹配 -> 0.0"""
fields = [
{"field_name": "id", "field_comment": "主键", "custom_comment": ""},
]
score = calc_keyword_score(
question="销售额",
table_name="users",
table_comment="用户表",
fields=fields
)
assert score == 0.0
def test_empty_question(self):
"""空问题 -> 0.0"""
score = calc_keyword_score(
question="",
table_name="sales_order",
table_comment="",
fields=[]
)
assert score == 0.0
def test_empty_keywords_after_split(self):
"""问题只有逗号/空格 -> 0.0"""
score = calc_keyword_score(
question=",, ,",
table_name="sales_order",
table_comment="",
fields=[]
)
assert score == 0.0
def test_priority_table_name_over_comment(self):
"""表名匹配应优先于注释匹配。"""
score = calc_keyword_score(
question="sales_order,销售订单",
table_name="sales_order",
table_comment="销售订单",
fields=[]
)
assert score == 1.0
def test_priority_comment_over_field_comment(self):
"""注释匹配(0.9)应优先于字段注释匹配(0.5)。"""
fields = [
{"field_name": "amt", "field_comment": "金额", "custom_comment": ""},
]
score = calc_keyword_score(
question="销售订单",
table_name="so_detail",
table_comment="销售订单",
fields=fields
)
assert score == 0.9
def test_priority_field_comment_over_field_name(self):
"""字段注释匹配(0.5)应优先于字段名匹配(0.3)。"""
fields = [
{"field_name": "销售额_column", "field_comment": "", "custom_comment": ""},
{"field_name": "amt", "field_comment": "销售额", "custom_comment": ""},
]
# 字段注释匹配 "销售额" -> 0.5
score = calc_keyword_score(
question="销售额",
table_name="detail",
table_comment="",
fields=fields
)
assert score == 0.5
def test_multiple_keywords_best_score_wins(self):
"""多个关键词时,返回最佳匹配分数。"""
fields = [
{"field_name": "qty", "field_comment": "数量", "custom_comment": ""},
]
# "sales_order" 匹配表名 -> 1.0
# "数量" 匹配字段注释 -> 0.5
# 最佳为 1.0
score = calc_keyword_score(
question="sales_order,数量",
table_name="sales_order",
table_comment="",
fields=fields
)
assert score == 1.0
def test_underscore_table_name_parts(self):
"""带下划线的表名应正确分割。"""
# "sales_order_detail" -> 部分: {"sales", "order", "detail"}
# "sales" 覆盖 1/3 -> 0.25 + 0.25*(1/3) ≈ 0.33
score = calc_keyword_score(
question="sales",
table_name="sales_order_detail",
table_comment="",
fields=[]
)
assert 0.3 <= score <= 0.35
def test_field_name_with_underscores(self):
"""带下划线的字段名应正确分割。"""
fields = [
{"field_name": "customer_order_count", "field_comment": "", "custom_comment": ""},
]
score = calc_keyword_score(
question="order",
table_name="stats",
table_comment="",
fields=fields
)
assert score == 0.3
def test_real_scenario_sales_query(self):
"""真实场景:'查询销售额' 应匹配包含 '销售额' 字段的表。"""
fields = [
{"field_name": "id", "field_comment": "主键", "custom_comment": ""},
{"field_name": "order_amount", "field_comment": "订单金额", "custom_comment": ""},
{"field_name": "sales_amount", "field_comment": "销售额", "custom_comment": ""},
]
score = calc_keyword_score(
question="销售额",
table_name="daily_report",
table_comment="日报表",
fields=fields
)
assert score == 0.5 # 字段注释匹配
def test_real_scenario_table_name_query(self):
"""真实场景:'查 sales_order 表最近的订单' -> 提取 'sales_order'。"""
score = calc_keyword_score(
question="sales_order,订单",
table_name="sales_order",
table_comment="销售订单",
fields=[]
)
assert score == 1.0 # 表名完整匹配
# =============================================================================
# expand_with_terminology 测试(mock 数据库)
# =============================================================================
class TestExpandWithTerminology:
"""术语同义词扩展测试。"""
def _make_term(self, id, word, pid=None, enabled=True, oid=1):
"""创建 mock 术语对象的辅助方法。"""
term = MagicMock()
term.id = id
term.word = word
term.pid = pid
term.enabled = enabled
term.oid = oid
return term
def test_expand_hit(self):
"""匹配术语的关键词应扩展同义词。"""
session = MagicMock()
session.query.return_value.filter.return_value.limit.return_value.all.return_value = [
self._make_term(1, "销售", pid=None),
self._make_term(2, "sales", pid=1),
self._make_term(3, "selling", pid=1),
]
result = expand_with_terminology("销售", session, oid=1)
keywords = [kw.strip() for kw in result.split(',')]
assert "销售" in keywords
assert "sales" in keywords
assert "selling" in keywords
def test_no_hit(self):
"""未匹配任何术语的关键词应保持不变。"""
session = MagicMock()
session.query.return_value.filter.return_value.limit.return_value.all.return_value = [
self._make_term(1, "客户", pid=None),
]
result = expand_with_terminology("销售额", session, oid=1)
assert result == "销售额"
def test_max_10_keywords(self):
"""扩展后的关键词总数不应超过 10 个。"""
session = MagicMock()
# 创建一个有多个同义词的术语
terms = [self._make_term(1, "销售", pid=None)]
for i in range(15):
terms.append(self._make_term(100 + i, f"synonym_{i}", pid=1))
session.query.return_value.filter.return_value.limit.return_value.all.return_value = terms
result = expand_with_terminology("销售", session, oid=1)
keywords = [kw.strip() for kw in result.split(',') if kw.strip()]
assert len(keywords) <= 10
def test_empty_terminology(self):
"""空术语表应返回原始关键词。"""
session = MagicMock()
session.query.return_value.filter.return_value.limit.return_value.all.return_value = []
result = expand_with_terminology("销售额,部门", session, oid=1)
assert result == "销售额,部门"
def test_none_session(self):
"""session 为 None 时应返回原始关键词。"""
result = expand_with_terminology("销售额", None, oid=1)
assert result == "销售额"
def test_none_oid(self):
"""oid 为 None 时应返回原始关键词。"""
session = MagicMock()
result = expand_with_terminology("销售额", session, oid=None)
assert result == "销售额"
def test_oid_zero_should_still_query(self):
"""oid=0 不应被视为假值 — 它是有效的组织 ID。"""
session = MagicMock()
session.query.return_value.filter.return_value.limit.return_value.all.return_value = [
self._make_term(1, "销售", pid=None, oid=0),
self._make_term(2, "revenue", pid=1, oid=0),
]
result = expand_with_terminology("销售", session, oid=0)
keywords = [kw.strip() for kw in result.split(',')]
assert "销售" in keywords
assert "revenue" in keywords
def test_empty_keywords(self):
"""空关键词字符串应原样返回。"""
session = MagicMock()
result = expand_with_terminology("", session, oid=1)
assert result == ""
def test_case_insensitive_match(self):
"""术语匹配应忽略大小写。"""
session = MagicMock()
session.query.return_value.filter.return_value.limit.return_value.all.return_value = [
self._make_term(1, "Sales", pid=None),
self._make_term(2, "revenue", pid=1),
]
result = expand_with_terminology("sales", session, oid=1)
keywords = [kw.strip() for kw in result.split(',')]
# "sales"(输入)匹配 "Sales"(术语),忽略大小写
# "Sales" 不会被添加,因为 "sales" 已在列表中(避免重复)
# "revenue"(子术语同义词)被添加
assert "sales" in keywords
assert "revenue" in keywords
assert len(keywords) == 2
def test_multiple_keywords_expansion(self):
"""多个关键词应各自独立扩展。"""
session = MagicMock()
session.query.return_value.filter.return_value.limit.return_value.all.return_value = [
self._make_term(1, "销售", pid=None),
self._make_term(2, "sales", pid=1),
self._make_term(3, "部门", pid=None),
self._make_term(4, "department", pid=3),
]
result = expand_with_terminology("销售,部门", session, oid=1)
keywords = [kw.strip() for kw in result.split(',')]
assert "sales" in keywords
assert "department" in keywords
def test_child_term_word_also_matches(self):
"""如果关键词匹配子术语的词,应扩展父术语和兄弟术语。"""
session = MagicMock()
session.query.return_value.filter.return_value.limit.return_value.all.return_value = [
self._make_term(1, "销售", pid=None),
self._make_term(2, "sales", pid=1),
self._make_term(3, "selling", pid=1),
]
# "sales" 匹配子术语,应扩展包含父术语 "销售" 和兄弟术语 "selling"
result = expand_with_terminology("sales", session, oid=1)
keywords = [kw.strip() for kw in result.split(',')]
assert "sales" in keywords
assert "销售" in keywords
assert "selling" in keywords
# =============================================================================
# calc_table_embedding 集成测试(mock embedding)
# =============================================================================
class TestCalcTableEmbedding:
"""融合评分管线集成测试。"""
def _make_table(self, id, table_name, schema_table, embedding, table_comment="", fields=None):
return {
"id": id,
"table_name": table_name,
"schema_table": schema_table,
"embedding": json.dumps(embedding),
"table_comment": table_comment,
"fields": fields or []
}
@patch('apps.datasource.embedding.table_embedding.settings')
@patch('apps.datasource.embedding.table_embedding.EmbeddingModelCache')
def test_exact_table_name_match_ranks_first(self, mock_embed_cache, mock_settings):
"""表名完整匹配关键词时,应排第一。"""
mock_settings.TABLE_EMBEDDING_KEYWORD_ENABLED = True
mock_settings.TABLE_EMBEDDING_ALPHA = 0.4
mock_settings.TABLE_EMBEDDING_COUNT = 10
mock_model = MagicMock()
mock_model.embed_query.return_value = [1.0, 0.0, 0.0]
mock_embed_cache.get_model.return_value = mock_model
tables = [
self._make_table(1, "customer", "# Table: customer", [0.0, 1.0, 0.0], "客户表",
[{"field_name": "name", "field_comment": "姓名", "custom_comment": ""}]),
self._make_table(2, "product", "# Table: product", [0.0, 0.0, 1.0], "产品表",
[{"field_name": "name", "field_comment": "产品名", "custom_comment": ""}]),
self._make_table(3, "sales_order", "# Table: sales_order", [0.9, 0.1, 0.0], "销售订单",
[{"field_name": "amount", "field_comment": "金额", "custom_comment": ""}]),
]
# 直接传入已提取的关键词
result = calc_table_embedding(tables, "查 sales_order 表最近的订单",
keywords="sales_order,订单")
# sales_order 应排第一,因为 keyword_score=1.0
assert result[0]['table_name'] == "sales_order"
assert result[0].get('keyword_score') == 1.0
@patch('apps.datasource.embedding.table_embedding.settings')
@patch('apps.datasource.embedding.table_embedding.EmbeddingModelCache')
def test_keyword_disabled_uses_vector_only(self, mock_embed_cache, mock_settings):
"""关键词匹配关闭时,应使用纯向量匹配。"""
mock_settings.TABLE_EMBEDDING_KEYWORD_ENABLED = False
mock_settings.TABLE_EMBEDDING_COUNT = 10
mock_model = MagicMock()
mock_model.embed_query.return_value = [1.0, 0.0, 0.0]
mock_embed_cache.get_model.return_value = mock_model
tables = [
self._make_table(1, "customer", "# Table: customer", [0.0, 1.0, 0.0]),
self._make_table(2, "sales_order", "# Table: sales_order", [0.9, 0.1, 0.0]),
]
result = calc_table_embedding(tables, "查 sales_order 表", keywords="sales_order,订单")
# 关键词匹配关闭,即使传入了 keywords 也应使用纯向量匹配
assert result[0]['table_name'] == "sales_order"
@patch('apps.datasource.embedding.table_embedding.settings')
@patch('apps.datasource.embedding.table_embedding.EmbeddingModelCache')
def test_no_keywords_falls_back_to_vector(self, mock_embed_cache, mock_settings):
"""未传入 keywords 时,应回退到纯向量匹配。"""
mock_settings.TABLE_EMBEDDING_KEYWORD_ENABLED = True
mock_settings.TABLE_EMBEDDING_COUNT = 10
mock_model = MagicMock()
mock_model.embed_query.return_value = [1.0, 0.0, 0.0]
mock_embed_cache.get_model.return_value = mock_model
tables = [
self._make_table(1, "customer", "# Table: customer", [0.0, 1.0, 0.0]),
self._make_table(2, "sales_order", "# Table: sales_order", [0.9, 0.1, 0.0]),
]
# 不传 keywords
result = calc_table_embedding(tables, "查 sales_order 表")
# 应回退到纯向量匹配
assert result[0]['table_name'] == "sales_order"
@patch('apps.datasource.embedding.table_embedding.settings')
@patch('apps.datasource.embedding.table_embedding.EmbeddingModelCache')
def test_multiple_exact_matches_all_rank_first(self, mock_embed_cache, mock_settings):
"""多个 keyword_score=1.0 的表都应排在其他表之前。"""
mock_settings.TABLE_EMBEDDING_KEYWORD_ENABLED = True
mock_settings.TABLE_EMBEDDING_ALPHA = 0.4
mock_settings.TABLE_EMBEDDING_COUNT = 10
mock_model = MagicMock()
mock_model.embed_query.return_value = [1.0, 0.0, 0.0]
mock_embed_cache.get_model.return_value = mock_model
tables = [
self._make_table(1, "product", "# Table: product", [0.0, 0.0, 1.0], "产品",
[{"field_name": "name", "field_comment": "名称", "custom_comment": ""}]),
self._make_table(2, "sales_order", "# Table: sales_order", [0.9, 0.1, 0.0], "",
[{"field_name": "amount", "field_comment": "", "custom_comment": ""}]),
self._make_table(3, "customer", "# Table: customer", [0.8, 0.2, 0.0], "",
[{"field_name": "name", "field_comment": "", "custom_comment": ""}]),
]
result = calc_table_embedding(tables, "查 sales_order 和 customer",
keywords="sales_order,customer")
# 前两个应是精确匹配(它们之间的顺序无所谓)
exact_match_names = {result[0]['table_name'], result[1]['table_name']}
assert exact_match_names == {"sales_order", "customer"}
# product 应排最后
assert result[2]['table_name'] == "product"
@patch('apps.datasource.embedding.table_embedding.settings')
@patch('apps.datasource.embedding.table_embedding.EmbeddingModelCache')
def test_fusion_score_formula(self, mock_embed_cache, mock_settings):
"""验证融合公式:final = α * vec + (1-α) * keyword。"""
mock_settings.TABLE_EMBEDDING_KEYWORD_ENABLED = True
mock_settings.TABLE_EMBEDDING_ALPHA = 0.4
mock_settings.TABLE_EMBEDDING_COUNT = 10
mock_model = MagicMock()
mock_model.embed_query.return_value = [1.0, 0.0]
mock_embed_cache.get_model.return_value = mock_model
# 表的 vec_score=0.8(通过余弦相似度),keyword_score 来自部分匹配
tables = [
self._make_table(1, "sales_order", "# Table: sales_order", [0.8, 0.6], "",
[{"field_name": "id", "field_comment": "", "custom_comment": ""}]),
]
result = calc_table_embedding(tables, "sales", keywords="sales")
# "sales" 部分匹配 "sales_order"(2 个部分中的 1 个)
# keyword_score = 0.25 + 0.25 * 0.5 = 0.375
# vec_score = cosine([1,0], [0.8, 0.6]) = 0.8 / (1 * 1) = 0.8
# final = 0.4 * 0.8 + 0.6 * 0.375 = 0.32 + 0.225 = 0.545
assert len(result) == 1
score = result[0]['cosine_similarity']
assert 0.5 <= score <= 0.6 # 允许小的浮点误差
@patch('apps.datasource.embedding.table_embedding.settings')
@patch('apps.datasource.embedding.table_embedding.EmbeddingModelCache')
def test_fallback_on_unexpected_error(self, mock_embed_cache, mock_settings):
"""遇到意外错误时,应回退到纯向量评分。"""
mock_settings.TABLE_EMBEDDING_KEYWORD_ENABLED = True
mock_settings.TABLE_EMBEDDING_ALPHA = 0.4
mock_settings.TABLE_EMBEDDING_COUNT = 10
# 让 embed_query 抛出意外错误
mock_model = MagicMock()
mock_model.embed_query.side_effect = RuntimeError("模型加载失败")
mock_embed_cache.get_model.return_value = mock_model
tables = [
self._make_table(1, "t1", "# Table: t1", [0.8, 0.6]),
self._make_table(2, "t2", "# Table: t2", [0.3, 0.7]),
]
# 不应抛出异常,应回退到纯向量匹配
result = calc_table_embedding(tables, "test", keywords="test")
assert len(result) == 2
if __name__ == '__main__':
pytest.main([__file__, '-v'])