From dc4a7380baa929e9aee5520dba59fea46ecb0617 Mon Sep 17 00:00:00 2001 From: ulleo Date: Thu, 28 Aug 2025 19:01:05 +0800 Subject: [PATCH] feat: terminology add oid --- .../versions/041_add_terminology_oid.py | 33 +++++++ backend/apps/chat/task/llm.py | 4 +- backend/apps/terminology/api/terminology.py | 13 +-- backend/apps/terminology/curd/terminology.py | 89 +++++++++++-------- .../terminology/models/terminology_model.py | 1 + backend/locales/en.json | 79 ++++++++-------- backend/locales/zh-CN.json | 80 +++++++++-------- backend/template.yaml | 26 +++++- 8 files changed, 201 insertions(+), 124 deletions(-) create mode 100644 backend/alembic/versions/041_add_terminology_oid.py diff --git a/backend/alembic/versions/041_add_terminology_oid.py b/backend/alembic/versions/041_add_terminology_oid.py new file mode 100644 index 00000000..1b4d783c --- /dev/null +++ b/backend/alembic/versions/041_add_terminology_oid.py @@ -0,0 +1,33 @@ +"""041_add_terminology_oid + +Revision ID: c4c3c36b720d +Revises: 0fc14c2cfe41 +Create Date: 2025-08-28 16:41:33.977242 + +""" +from alembic import op +import sqlalchemy as sa +import sqlmodel.sql.sqltypes +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'c4c3c36b720d' +down_revision = '0fc14c2cfe41' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + + op.add_column('terminology', sa.Column('oid', sa.BigInteger(), nullable=True)) + + op.execute('update terminology set oid=1 where oid is null') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('terminology', 'oid') + + # ### end Alembic commands ### diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index 06dcb3b2..0ed0b78a 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -220,7 +220,7 @@ class LLMService: self.chat_question.data = orjson.dumps(data.get('data')).decode() analysis_msg: List[Union[BaseMessage, dict[str, Any]]] = [] - self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question) + self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question, self.current_user.oid) analysis_msg.append(SystemMessage(content=self.chat_question.analysis_sys_question())) analysis_msg.append(HumanMessage(content=self.chat_question.analysis_user_question())) @@ -894,7 +894,7 @@ class LLMService: def run_task(self, in_chat: bool = True): try: - self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question) + self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question, self.current_user.oid) self.init_messages() # return id diff --git a/backend/apps/terminology/api/terminology.py b/backend/apps/terminology/api/terminology.py index 2467db3a..edb0df99 100644 --- a/backend/apps/terminology/api/terminology.py +++ b/backend/apps/terminology/api/terminology.py @@ -5,15 +5,15 @@ from fastapi import APIRouter, Query from apps.terminology.curd.terminology import page_terminology, create_terminology, update_terminology, \ delete_terminology from apps.terminology.models.terminology_model import TerminologyInfo -from common.core.deps import SessionDep +from common.core.deps import SessionDep, CurrentUser, Trans router = APIRouter(tags=["Terminology"], prefix="/system/terminology") @router.get("/page/{current_page}/{page_size}") -async def pager(session: SessionDep, current_page: int, page_size: int, +async def pager(session: SessionDep, current_user: CurrentUser, current_page: int, page_size: int, word: Optional[str] = Query(None, description="搜索术语(可选)")): - current_page, page_size, total_count, total_pages, _list = page_terminology(session, current_page, page_size, word) + current_page, page_size, total_count, total_pages, _list = page_terminology(session, current_page, page_size, word, current_user.oid) return { "current_page": current_page, @@ -25,11 +25,12 @@ async def pager(session: SessionDep, current_page: int, page_size: int, @router.put("") -async def create_or_update(session: SessionDep, info: TerminologyInfo): +async def create_or_update(session: SessionDep, current_user: CurrentUser, trans: Trans, info: TerminologyInfo): + oid = current_user.oid if info.id: - return update_terminology(session, info) + return update_terminology(session, info, oid, trans) else: - return create_terminology(session, info) + return create_terminology(session, info, oid) @router.delete("") diff --git a/backend/apps/terminology/curd/terminology.py b/backend/apps/terminology/curd/terminology.py index 97dc564b..ae710f8a 100644 --- a/backend/apps/terminology/curd/terminology.py +++ b/backend/apps/terminology/curd/terminology.py @@ -15,12 +15,13 @@ from apps.ai_model.embedding import EmbeddingModelCache from apps.template.generate_chart.generator import get_base_terminology_template from apps.terminology.models.terminology_model import Terminology, TerminologyInfo from common.core.config import settings -from common.core.deps import SessionDep +from common.core.deps import SessionDep, Trans executor = ThreadPoolExecutor(max_workers=200) -def page_terminology(session: SessionDep, current_page: int = 1, page_size: int = 10, name: Optional[str] = None): +def page_terminology(session: SessionDep, current_page: int = 1, page_size: int = 10, name: Optional[str] = None, + oid: Optional[int] = 1): _list: List[TerminologyInfo] = [] child = aliased(Terminology) @@ -91,7 +92,7 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int children_subquery, Terminology.id == children_subquery.c.pid ) - .where(Terminology.id.in_(paginated_parent_ids)) + .where(and_(Terminology.id.in_(paginated_parent_ids), Terminology.oid == oid)) .order_by(Terminology.create_time.desc()) ) else: @@ -120,7 +121,7 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int func.jsonb_agg(child.word).filter(child.word.isnot(None)).label('other_words') ) .outerjoin(child, and_(Terminology.id == child.pid)) - .where(Terminology.id.in_(paginated_parent_ids)) + .where(and_(Terminology.id.in_(paginated_parent_ids), Terminology.oid == oid)) .group_by(Terminology.id, Terminology.word) .order_by(Terminology.create_time.desc()) ) @@ -139,9 +140,9 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int return current_page, page_size, total_count, total_pages, _list -def create_terminology(session: SessionDep, info: TerminologyInfo): +def create_terminology(session: SessionDep, info: TerminologyInfo, oid: int): create_time = datetime.datetime.now() - parent = Terminology(word=info.word, create_time=create_time, description=info.description) + parent = Terminology(word=info.word, create_time=create_time, description=info.description, oid=oid) result = Terminology(**parent.model_dump()) @@ -158,7 +159,7 @@ def create_terminology(session: SessionDep, info: TerminologyInfo): if other_word.strip() == "": continue _list.append( - Terminology(pid=result.id, word=other_word, create_time=create_time)) + Terminology(pid=result.id, word=other_word, create_time=create_time, oid=oid)) session.bulk_save_objects(_list) session.flush() session.commit() @@ -169,7 +170,14 @@ def create_terminology(session: SessionDep, info: TerminologyInfo): return result.id -def update_terminology(session: SessionDep, info: TerminologyInfo): +def update_terminology(session: SessionDep, info: TerminologyInfo, oid: int, trans: Trans): + count = session.query(Terminology).filter( + Terminology.oid == oid, + Terminology.id == info.id + ).count() + if count == 0: + raise Exception(trans('i18n_terminology.terminology_not_exists')) + stmt = update(Terminology).where(and_(Terminology.id == info.id)).values( word=info.word, description=info.description, @@ -188,7 +196,7 @@ def update_terminology(session: SessionDep, info: TerminologyInfo): if other_word.strip() == "": continue _list.append( - Terminology(pid=info.id, word=other_word, create_time=create_time)) + Terminology(pid=info.id, word=other_word, create_time=create_time, oid=oid)) session.bulk_save_objects(_list) session.flush() session.commit() @@ -196,6 +204,7 @@ def update_terminology(session: SessionDep, info: TerminologyInfo): # embedding run_save_embeddings([info.id]) + return info.id @@ -256,23 +265,19 @@ def save_embeddings(ids: List[int]): embedding_sql = f""" -SELECT id, pid, word, description, similarity +SELECT id, pid, word, similarity FROM -(SELECT id, pid, word, -COALESCE( - description, - (SELECT description FROM terminology AS parent WHERE parent.id = child.pid) - ) AS description, +(SELECT id, pid, word, oid, ( 1 - (embedding <=> :embedding_array) ) AS similarity FROM terminology AS child ) TEMP -WHERE similarity > {settings.EMBEDDING_SIMILARITY} +WHERE similarity > {settings.EMBEDDING_SIMILARITY} and oid = :oid ORDER BY similarity DESC LIMIT {settings.EMBEDDING_TOP_COUNT} """ -def select_terminology_by_word(session: SessionDep, word: str): +def select_terminology_by_word(session: SessionDep, word: str, oid: int): if word.strip() == "": return [] @@ -283,22 +288,16 @@ def select_terminology_by_word(session: SessionDep, word: str): Terminology.id, Terminology.pid, Terminology.word, - func.coalesce( - Terminology.description, - select(Terminology.description) - .where(and_(Terminology.id == Terminology.pid)) - .scalar_subquery() - ).label('description') ) .where( - text(":sentence ILIKE '%' || word || '%'") + and_(text(":sentence ILIKE '%' || word || '%'"), Terminology.oid == oid) ) ) results = session.execute(stmt, {'sentence': word}).fetchall() for row in results: - _list.append(Terminology(id=row.id, word=row.word, pid=row.pid, description=row.description)) + _list.append(Terminology(id=row.id, word=row.word, pid=row.pid)) if settings.EMBEDDING_ENABLED: try: @@ -306,25 +305,31 @@ def select_terminology_by_word(session: SessionDep, word: str): embedding = model.embed_query(word) - print(embedding_sql) - results = session.execute(text(embedding_sql), {'embedding_array': str(embedding)}) + results = session.execute(text(embedding_sql), {'embedding_array': str(embedding), 'oid': oid}) for row in results: - _list.append(Terminology(id=row.id, word=row.word, pid=row.pid, description=row.description)) + _list.append(Terminology(id=row.id, word=row.word, pid=row.pid)) except Exception: traceback.print_exc() _map: dict = {} - _ids: set[int] = set() + _ids: list[int] = [] for row in _list: - if row.id in _ids: + if row.id in _ids or row.pid in _ids: continue - _ids.add(row.id) - if row.pid: - pid = str(row.pid) + if row.pid is not None: + _ids.append(row.pid) else: - pid = str(row.id) + _ids.append(row.id) + + if len(_ids) == 0: + return [] + + t_list = session.query(Terminology.id, Terminology.pid, Terminology.word, Terminology.description).filter( + or_(Terminology.id.in_(_ids), Terminology.pid.in_(_ids))).all() + for row in t_list: + pid = str(row.pid) if row.pid is not None else str(row.id) if _map.get(pid) is None: _map[pid] = {'words': [], 'description': row.description} _map[pid]['words'].append(row.word) @@ -350,6 +355,7 @@ def to_xml_string(_dict: list[dict] | dict, root: str = 'terminologies') -> str: item_name_func = lambda x: 'terminology' if x == 'terminologies' else 'word' if x == 'words' else 'item' dicttoxml.LOG.setLevel(logging.ERROR) xml = dicttoxml.dicttoxml(_dict, + cdata=['word', 'description'], custom_root=root, item_func=item_name_func, xml_declaration=False, @@ -361,11 +367,22 @@ def to_xml_string(_dict: list[dict] | dict, root: str = 'terminologies') -> str: end_index = pretty_xml.find('>') + 1 pretty_xml = pretty_xml[end_index:].lstrip() + # 替换所有 XML 转义字符 + escape_map = { + '<': '<', + '>': '>', + '&': '&', + '"': '"', + ''': "'" + } + for escaped, original in escape_map.items(): + pretty_xml = pretty_xml.replace(escaped, original) + return pretty_xml -def get_terminology_template(session: SessionDep, question: str) -> str: - _results = select_terminology_by_word(session, question) +def get_terminology_template(session: SessionDep, question: str, oid: int) -> str: + _results = select_terminology_by_word(session, question, oid) if _results and len(_results) > 0: terminology = to_xml_string(_results) template = get_base_terminology_template().format(terminologies=terminology) diff --git a/backend/apps/terminology/models/terminology_model.py b/backend/apps/terminology/models/terminology_model.py index d467cc5c..57c35ce4 100644 --- a/backend/apps/terminology/models/terminology_model.py +++ b/backend/apps/terminology/models/terminology_model.py @@ -10,6 +10,7 @@ from sqlmodel import SQLModel, Field class Terminology(SQLModel, table=True): __tablename__ = "terminology" id: Optional[int] = Field(sa_column=Column(BigInteger, Identity(always=True), primary_key=True)) + oid: Optional[int] = Field(sa_column=Column(BigInteger, nullable=True, default=1)) pid: Optional[int] = Field(sa_column=Column(BigInteger, nullable=True)) create_time: Optional[datetime] = Field(sa_column=Column(DateTime(timezone=False), nullable=True)) word: Optional[str] = Field(max_length=255) diff --git a/backend/locales/en.json b/backend/locales/en.json index acee9415..429be3a4 100644 --- a/backend/locales/en.json +++ b/backend/locales/en.json @@ -1,40 +1,43 @@ { - "i18n_default_workspace": "Default workspace", - "i18n_ds_name_exist": "Name already exists", - "i18n_concat_admin": "Please contact administrator!", - "i18n_exist": "{msg} already exists!", - "i18n_not_exist": "{msg} not exists", - "i18n_error": "{key} error!", - "i18n_miss_args": "Missing {key} parameter!", - "i18n_format_invalid": "{key} format is incorrect!", - "i18n_login": { - "account_pwd_error": "Account or password error!", - "no_associated_ws": "No associated workspace, {msg}", - "user_disable": "Account disabled, {msg}" - }, - "i18n_user": { - "account": "Account", - "email": "Email", - "password": "Password", - "language_not_support": "System does not support [{key}] language!", - "ws_miss": "The current user is not in the workspace [{ws}]!" - }, - "i18n_ws": { - "title": "Workspace" - }, - "i18n_permission": { - "only_admin": "Only administrators can call this!", - "no_permission": "No permission to access {url}{msg}", - "authenticate_invalid": "Authenticate invalid [{msg}]", - "token_expired": "Token has expired" - }, - "i18n_llm": { - "validate_error": "Validation failed [{msg}]", - "delete_default_error": "Cannot delete default model [{key}]!", - "miss_default": "The default large language model has not been configured" - }, - "i18n_ds_invalid": "Datasource Invalid", - "i18n_embedded": { - "invalid_origin": "Domain verification failed [{origin}]" - } + "i18n_default_workspace": "Default workspace", + "i18n_ds_name_exist": "Name already exists", + "i18n_concat_admin": "Please contact administrator!", + "i18n_exist": "{msg} already exists!", + "i18n_not_exist": "{msg} not exists", + "i18n_error": "{key} error!", + "i18n_miss_args": "Missing {key} parameter!", + "i18n_format_invalid": "{key} format is incorrect!", + "i18n_login": { + "account_pwd_error": "Account or password error!", + "no_associated_ws": "No associated workspace, {msg}", + "user_disable": "Account disabled, {msg}" + }, + "i18n_user": { + "account": "Account", + "email": "Email", + "password": "Password", + "language_not_support": "System does not support [{key}] language!", + "ws_miss": "The current user is not in the workspace [{ws}]!" + }, + "i18n_ws": { + "title": "Workspace" + }, + "i18n_permission": { + "only_admin": "Only administrators can call this!", + "no_permission": "No permission to access {url}{msg}", + "authenticate_invalid": "Authenticate invalid [{msg}]", + "token_expired": "Token has expired" + }, + "i18n_llm": { + "validate_error": "Validation failed [{msg}]", + "delete_default_error": "Cannot delete default model [{key}]!", + "miss_default": "The default large language model has not been configured" + }, + "i18n_ds_invalid": "Datasource Invalid", + "i18n_embedded": { + "invalid_origin": "Domain verification failed [{origin}]" + }, + "i18n_terminology": { + "terminology_not_exists": "Terminology does not exists" + } } \ No newline at end of file diff --git a/backend/locales/zh-CN.json b/backend/locales/zh-CN.json index 77ea01aa..70a8f171 100644 --- a/backend/locales/zh-CN.json +++ b/backend/locales/zh-CN.json @@ -1,41 +1,43 @@ { - "i18n_default_workspace": "默认工作空间", - "i18n_ds_name_exist": "名称已存在", - "i18n_concat_admin": "请联系管理员！", - "i18n_exist": "{msg}已存在！", - "i18n_not_exist": "{msg}不存在！", - "i18n_error": "{key}错误！", - "i18n_miss_args": "缺失{key}参数！", - "i18n_format_invalid": "{key}格式不正确！", - "i18n_login": { - "account_pwd_error": "账号或密码错误！", - "no_associated_ws": "没有关联的工作空间，{msg}", - "user_disable": "账号已禁用，{msg}" - }, - "i18n_user": { - "account": "账号", - "email": "邮箱", - "password": "密码", - "language_not_support": "系统不支持[{key}]语言！", - "ws_miss": "当前用户不在工作空间[{ws}]中！" - }, - "i18n_ws": { - "title": "工作空间" - }, - "i18n_permission": { - "only_admin": "仅支持管理员调用！", - "no_permission": "无权调用{url}{msg}", - "authenticate_invalid": "认证无效【{msg}】", - "token_expired": "Token 已过期" - - }, - "i18n_llm": { - "validate_error": "校验失败[{msg}]", - "delete_default_error": "无法删除默认模型[{key}]！", - "miss_default": "尚未配置默认大语言模型" - }, - "i18n_ds_invalid": "数据源连接无效", - "i18n_embedded": { - "invalid_origin": "域名校验失败【{origin}】" - } + "i18n_default_workspace": "默认工作空间", + "i18n_ds_name_exist": "名称已存在", + "i18n_concat_admin": "请联系管理员！", + "i18n_exist": "{msg}已存在！", + "i18n_not_exist": "{msg}不存在！", + "i18n_error": "{key}错误！", + "i18n_miss_args": "缺失{key}参数！", + "i18n_format_invalid": "{key}格式不正确！", + "i18n_login": { + "account_pwd_error": "账号或密码错误！", + "no_associated_ws": "没有关联的工作空间，{msg}", + "user_disable": "账号已禁用，{msg}" + }, + "i18n_user": { + "account": "账号", + "email": "邮箱", + "password": "密码", + "language_not_support": "系统不支持[{key}]语言！", + "ws_miss": "当前用户不在工作空间[{ws}]中！" + }, + "i18n_ws": { + "title": "工作空间" + }, + "i18n_permission": { + "only_admin": "仅支持管理员调用！", + "no_permission": "无权调用{url}{msg}", + "authenticate_invalid": "认证无效【{msg}】", + "token_expired": "Token 已过期" + }, + "i18n_llm": { + "validate_error": "校验失败[{msg}]", + "delete_default_error": "无法删除默认模型[{key}]！", + "miss_default": "尚未配置默认大语言模型" + }, + "i18n_ds_invalid": "数据源连接无效", + "i18n_embedded": { + "invalid_origin": "域名校验失败【{origin}】" + }, + "i18n_terminology": { + "terminology_not_exists": "该术语不存在" + } } \ No newline at end of file diff --git a/backend/template.yaml b/backend/template.yaml index 37a48e09..d03cd141 100644 --- a/backend/template.yaml +++ b/backend/template.yaml @@ -12,7 +12,7 @@ template: 内有等信息； 其中，：提供数据库引擎及版本信息； ：以 M-Schema 格式提供数据库表结构信息； - ：提供一组专业术语，块内每一个就是专业术语，其中同一个内的多个代表术语的多种叫法，也就是术语与它的同义词，即该专业术语对应的专业术语的描述，其中也可能是能够用来参考的计算公式 + ：提供一组专业术语，块内每一个就是专业术语，其中同一个内的多个代表术语的多种叫法，也就是术语与它的同义词，即该专业术语对应的专业术语的描述，其中也可能是能够用来参考的计算公式，或者是一些其他的查询条件 用户的提问在内，内则会提供上次执行你提供的SQL时会出现的错误信息，内的会告诉你用户当前提问的时间 @@ -128,6 +128,13 @@ template: 指在一个季度或一年，一个国家或地区的经济中所生产出的全部最终产品和劳务的价值。 + + + 中国 + 中国大陆 + + 查询SQL时若作为查询条件，将"中国"作为查询用的值 + @@ -166,7 +173,7 @@ template: 查询各个国家每年的GDP - {{"success":true,"sql":"SELECT \"country\" AS \"country_alias\", \"continent\" AS \"continent_alias\", \"year\" AS \"year_alias\", \"gdp\" AS \"gdp_alias\" FROM \"Sample_Database\".\"sample_country_gdp\" ORDER BY \"country\", \"year\" LIMIT 1000","tables":["sample_country_gdp"],"chart-type":"line"}} + {{"success":true,"sql":"SELECT \"country\" AS \"country_name\", \"continent\" AS \"continent_name\", \"year\" AS \"year\", \"gdp\" AS \"gdp\" FROM \"Sample_Database\".\"sample_country_gdp\" ORDER BY \"country\", \"year\" LIMIT 1000","tables":["sample_country_gdp"],"chart-type":"line"}} @@ -178,7 +185,20 @@ template: 使用饼图展示去年各个国家的GDP - {{"success":true,"sql":"SELECT \"country\" AS \"country_alias\", \"gdp\" AS \"gdp_alias\" FROM \"Sample_Database\".\"sample_country_gdp\" WHERE \"year\" = '2024' ORDER BY \"gdp\" DESC LIMIT 1000","tables":["sample_country_gdp"],"chart-type":"pie"}} + {{"success":true,"sql":"SELECT \"country\" AS \"country_name\", \"gdp\" AS \"gdp\" FROM \"Sample_Database\".\"sample_country_gdp\" WHERE \"year\" = '2024' ORDER BY \"gdp\" DESC LIMIT 1000","tables":["sample_country_gdp"],"chart-type":"pie"}} + + + + + + + + 2025-08-08 11:24:00 + + + 查询今年中国大陆的GDP + + {{"success":true,"sql":"SELECT \"country\" AS \"country_name\", \"gdp\" AS \"gdp\" FROM \"Sample_Database\".\"sample_country_gdp\" WHERE \"year\" = '2025' AND \"国家\" = '中国' LIMIT 1000","tables":["sample_country_gdp"],"chart-type":"table"}}