klinsmaya/SQLBot
1
0
Fork 0
mirror of https://gh-proxy.org/https://github.com/klinsmaya/SQLBot.git synced 2026-11-08 01:06:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: terminology add oid

Browse Source
This commit is contained in:
ulleo
2025-08-28 19:01:05 +08:00
parent 40699d29b4
commit dc4a7380ba
8 changed files with 201 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/alembic/versions/041_add_terminology_oid.py
+33
View File
@@ -0,0 +1,33 @@
"""041_add_terminology_oid
Revision ID: c4c3c36b720d
Revises: 0fc14c2cfe41
Create Date: 2025-08-28 16:41:33.977242
"""
from alembic import op
import sqlalchemy as sa
import sqlmodel.sql.sqltypes
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = 'c4c3c36b720d'
down_revision = '0fc14c2cfe41'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('terminology', sa.Column('oid', sa.BigInteger(), nullable=True))
op.execute('update terminology set oid=1 where oid is null')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('terminology', 'oid')
# ### end Alembic commands ###
backend/apps/chat/task/llm.py
+2 -2
View File
@@ -220,7 +220,7 @@ class LLMService:
self.chat_question.data = orjson.dumps(data.get('data')).decode()
analysis_msg: List[Union[BaseMessage, dict[str, Any]]] = []
self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question)
self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question, self.current_user.oid)
analysis_msg.append(SystemMessage(content=self.chat_question.analysis_sys_question()))
analysis_msg.append(HumanMessage(content=self.chat_question.analysis_user_question()))
@@ -894,7 +894,7 @@ class LLMService:
def run_task(self, in_chat: bool = True):
try:
self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question)
self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question, self.current_user.oid)
self.init_messages()
# return id
backend/apps/terminology/api/terminology.py
+7 -6
View File
@@ -5,15 +5,15 @@ from fastapi import APIRouter, Query
from apps.terminology.curd.terminology import page_terminology, create_terminology, update_terminology, \
delete_terminology
from apps.terminology.models.terminology_model import TerminologyInfo
from common.core.deps import SessionDep
from common.core.deps import SessionDep, CurrentUser, Trans
router = APIRouter(tags=["Terminology"], prefix="/system/terminology")
@router.get("/page/{current_page}/{page_size}")
async def pager(session: SessionDep, current_page: int, page_size: int,
async def pager(session: SessionDep, current_user: CurrentUser, current_page: int, page_size: int,
word: Optional[str] = Query(None, description="搜索术语(可选)")):
current_page, page_size, total_count, total_pages, _list = page_terminology(session, current_page, page_size, word)
current_page, page_size, total_count, total_pages, _list = page_terminology(session, current_page, page_size, word, current_user.oid)
return {
"current_page": current_page,
@@ -25,11 +25,12 @@ async def pager(session: SessionDep, current_page: int, page_size: int,
@router.put("")
async def create_or_update(session: SessionDep, info: TerminologyInfo):
async def create_or_update(session: SessionDep, current_user: CurrentUser, trans: Trans, info: TerminologyInfo):
oid = current_user.oid
if info.id:
return update_terminology(session, info)
return update_terminology(session, info, oid, trans)
else:
return create_terminology(session, info)
return create_terminology(session, info, oid)
@router.delete("")
backend/apps/terminology/curd/terminology.py
+53 -36
View File
@@ -15,12 +15,13 @@ from apps.ai_model.embedding import EmbeddingModelCache
from apps.template.generate_chart.generator import get_base_terminology_template
from apps.terminology.models.terminology_model import Terminology, TerminologyInfo
from common.core.config import settings
from common.core.deps import SessionDep
from common.core.deps import SessionDep, Trans
executor = ThreadPoolExecutor(max_workers=200)
def page_terminology(session: SessionDep, current_page: int = 1, page_size: int = 10, name: Optional[str] = None):
def page_terminology(session: SessionDep, current_page: int = 1, page_size: int = 10, name: Optional[str] = None,
oid: Optional[int] = 1):
_list: List[TerminologyInfo] = []
child = aliased(Terminology)
@@ -91,7 +92,7 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int
children_subquery,
Terminology.id == children_subquery.c.pid
)
.where(Terminology.id.in_(paginated_parent_ids))
.where(and_(Terminology.id.in_(paginated_parent_ids), Terminology.oid == oid))
.order_by(Terminology.create_time.desc())
)
else:
@@ -120,7 +121,7 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int
func.jsonb_agg(child.word).filter(child.word.isnot(None)).label('other_words')
)
.outerjoin(child, and_(Terminology.id == child.pid))
.where(Terminology.id.in_(paginated_parent_ids))
.where(and_(Terminology.id.in_(paginated_parent_ids), Terminology.oid == oid))
.group_by(Terminology.id, Terminology.word)
.order_by(Terminology.create_time.desc())
)
@@ -139,9 +140,9 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int
return current_page, page_size, total_count, total_pages, _list
def create_terminology(session: SessionDep, info: TerminologyInfo):
def create_terminology(session: SessionDep, info: TerminologyInfo, oid: int):
create_time = datetime.datetime.now()
parent = Terminology(word=info.word, create_time=create_time, description=info.description)
parent = Terminology(word=info.word, create_time=create_time, description=info.description, oid=oid)
result = Terminology(**parent.model_dump())
@@ -158,7 +159,7 @@ def create_terminology(session: SessionDep, info: TerminologyInfo):
if other_word.strip() == "":
continue
_list.append(
Terminology(pid=result.id, word=other_word, create_time=create_time))
Terminology(pid=result.id, word=other_word, create_time=create_time, oid=oid))
session.bulk_save_objects(_list)
session.flush()
session.commit()
@@ -169,7 +170,14 @@ def create_terminology(session: SessionDep, info: TerminologyInfo):
return result.id
def update_terminology(session: SessionDep, info: TerminologyInfo):
def update_terminology(session: SessionDep, info: TerminologyInfo, oid: int, trans: Trans):
count = session.query(Terminology).filter(
Terminology.oid == oid,
Terminology.id == info.id
).count()
if count == 0:
raise Exception(trans('i18n_terminology.terminology_not_exists'))
stmt = update(Terminology).where(and_(Terminology.id == info.id)).values(
word=info.word,
description=info.description,
@@ -188,7 +196,7 @@ def update_terminology(session: SessionDep, info: TerminologyInfo):
if other_word.strip() == "":
continue
_list.append(
Terminology(pid=info.id, word=other_word, create_time=create_time))
Terminology(pid=info.id, word=other_word, create_time=create_time, oid=oid))
session.bulk_save_objects(_list)
session.flush()
session.commit()
@@ -196,6 +204,7 @@ def update_terminology(session: SessionDep, info: TerminologyInfo):
# embedding
run_save_embeddings([info.id])
return info.id
@@ -256,23 +265,19 @@ def save_embeddings(ids: List[int]):
embedding_sql = f"""
SELECT id, pid, word, description, similarity
SELECT id, pid, word, similarity
FROM
(SELECT id, pid, word,
COALESCE(
description,
(SELECT description FROM terminology AS parent WHERE parent.id = child.pid)
) AS description,
(SELECT id, pid, word, oid,
( 1 - (embedding <=> :embedding_array) ) AS similarity
FROM terminology AS child
) TEMP
WHERE similarity > {settings.EMBEDDING_SIMILARITY}
WHERE similarity > {settings.EMBEDDING_SIMILARITY} and oid = :oid
ORDER BY similarity DESC
LIMIT {settings.EMBEDDING_TOP_COUNT}
"""
def select_terminology_by_word(session: SessionDep, word: str):
def select_terminology_by_word(session: SessionDep, word: str, oid: int):
if word.strip() == "":
return []
@@ -283,22 +288,16 @@ def select_terminology_by_word(session: SessionDep, word: str):
Terminology.id,
Terminology.pid,
Terminology.word,
func.coalesce(
Terminology.description,
select(Terminology.description)
.where(and_(Terminology.id == Terminology.pid))
.scalar_subquery()
).label('description')
)
.where(
text(":sentence ILIKE '%' || word || '%'")
and_(text(":sentence ILIKE '%' || word || '%'"), Terminology.oid == oid)
)
)
results = session.execute(stmt, {'sentence': word}).fetchall()
for row in results:
_list.append(Terminology(id=row.id, word=row.word, pid=row.pid, description=row.description))
_list.append(Terminology(id=row.id, word=row.word, pid=row.pid))
if settings.EMBEDDING_ENABLED:
try:
@@ -306,25 +305,31 @@ def select_terminology_by_word(session: SessionDep, word: str):
embedding = model.embed_query(word)
print(embedding_sql)
results = session.execute(text(embedding_sql), {'embedding_array': str(embedding)})
results = session.execute(text(embedding_sql), {'embedding_array': str(embedding), 'oid': oid})
for row in results:
_list.append(Terminology(id=row.id, word=row.word, pid=row.pid, description=row.description))
_list.append(Terminology(id=row.id, word=row.word, pid=row.pid))
except Exception:
traceback.print_exc()
_map: dict = {}
_ids: set[int] = set()
_ids: list[int] = []
for row in _list:
if row.id in _ids:
if row.id in _ids or row.pid in _ids:
continue
_ids.add(row.id)
if row.pid:
pid = str(row.pid)
if row.pid is not None:
_ids.append(row.pid)
else:
pid = str(row.id)
_ids.append(row.id)
if len(_ids) == 0:
return []
t_list = session.query(Terminology.id, Terminology.pid, Terminology.word, Terminology.description).filter(
or_(Terminology.id.in_(_ids), Terminology.pid.in_(_ids))).all()
for row in t_list:
pid = str(row.pid) if row.pid is not None else str(row.id)
if _map.get(pid) is None:
_map[pid] = {'words': [], 'description': row.description}
_map[pid]['words'].append(row.word)
@@ -350,6 +355,7 @@ def to_xml_string(_dict: list[dict] | dict, root: str = 'terminologies') -> str:
item_name_func = lambda x: 'terminology' if x == 'terminologies' else 'word' if x == 'words' else 'item'
dicttoxml.LOG.setLevel(logging.ERROR)
xml = dicttoxml.dicttoxml(_dict,
cdata=['word', 'description'],
custom_root=root,
item_func=item_name_func,
xml_declaration=False,
@@ -361,11 +367,22 @@ def to_xml_string(_dict: list[dict] | dict, root: str = 'terminologies') -> str:
end_index = pretty_xml.find('>') + 1
pretty_xml = pretty_xml[end_index:].lstrip()
# 替换所有 XML 转义字符
escape_map = {
'&lt;': '<',
'&gt;': '>',
'&amp;': '&',
'&quot;': '"',
'&apos;': "'"
}
for escaped, original in escape_map.items():
pretty_xml = pretty_xml.replace(escaped, original)
return pretty_xml
def get_terminology_template(session: SessionDep, question: str) -> str:
_results = select_terminology_by_word(session, question)
def get_terminology_template(session: SessionDep, question: str, oid: int) -> str:
_results = select_terminology_by_word(session, question, oid)
if _results and len(_results) > 0:
terminology = to_xml_string(_results)
template = get_base_terminology_template().format(terminologies=terminology)
backend/apps/terminology/models/terminology_model.py
+1
View File
@@ -10,6 +10,7 @@ from sqlmodel import SQLModel, Field
class Terminology(SQLModel, table=True):
__tablename__ = "terminology"
id: Optional[int] = Field(sa_column=Column(BigInteger, Identity(always=True), primary_key=True))
oid: Optional[int] = Field(sa_column=Column(BigInteger, nullable=True, default=1))
pid: Optional[int] = Field(sa_column=Column(BigInteger, nullable=True))
create_time: Optional[datetime] = Field(sa_column=Column(DateTime(timezone=False), nullable=True))
word: Optional[str] = Field(max_length=255)
backend/locales/en.json
+41 -38
View File
@@ -1,40 +1,43 @@
{
"i18n_default_workspace": "Default workspace",
"i18n_ds_name_exist": "Name already exists",
"i18n_concat_admin": "Please contact administrator!",
"i18n_exist": "{msg} already exists!",
"i18n_not_exist": "{msg} not exists",
"i18n_error": "{key} error!",
"i18n_miss_args": "Missing {key} parameter!",
"i18n_format_invalid": "{key} format is incorrect!",
"i18n_login": {
"account_pwd_error": "Account or password error!",
"no_associated_ws": "No associated workspace, {msg}",
"user_disable": "Account disabled, {msg}"
},
"i18n_user": {
"account": "Account",
"email": "Email",
"password": "Password",
"language_not_support": "System does not support [{key}] language!",
"ws_miss": "The current user is not in the workspace [{ws}]!"
},
"i18n_ws": {
"title": "Workspace"
},
"i18n_permission": {
"only_admin": "Only administrators can call this!",
"no_permission": "No permission to access {url}{msg}",
"authenticate_invalid": "Authenticate invalid [{msg}]",
"token_expired": "Token has expired"
},
"i18n_llm": {
"validate_error": "Validation failed [{msg}]",
"delete_default_error": "Cannot delete default model [{key}]!",
"miss_default": "The default large language model has not been configured"
},
"i18n_ds_invalid": "Datasource Invalid",
"i18n_embedded": {
"invalid_origin": "Domain verification failed [{origin}]"
}
"i18n_default_workspace": "Default workspace",
"i18n_ds_name_exist": "Name already exists",
"i18n_concat_admin": "Please contact administrator!",
"i18n_exist": "{msg} already exists!",
"i18n_not_exist": "{msg} not exists",
"i18n_error": "{key} error!",
"i18n_miss_args": "Missing {key} parameter!",
"i18n_format_invalid": "{key} format is incorrect!",
"i18n_login": {
"account_pwd_error": "Account or password error!",
"no_associated_ws": "No associated workspace, {msg}",
"user_disable": "Account disabled, {msg}"
},
"i18n_user": {
"account": "Account",
"email": "Email",
"password": "Password",
"language_not_support": "System does not support [{key}] language!",
"ws_miss": "The current user is not in the workspace [{ws}]!"
},
"i18n_ws": {
"title": "Workspace"
},
"i18n_permission": {
"only_admin": "Only administrators can call this!",
"no_permission": "No permission to access {url}{msg}",
"authenticate_invalid": "Authenticate invalid [{msg}]",
"token_expired": "Token has expired"
},
"i18n_llm": {
"validate_error": "Validation failed [{msg}]",
"delete_default_error": "Cannot delete default model [{key}]!",
"miss_default": "The default large language model has not been configured"
},
"i18n_ds_invalid": "Datasource Invalid",
"i18n_embedded": {
"invalid_origin": "Domain verification failed [{origin}]"
},
"i18n_terminology": {
"terminology_not_exists": "Terminology does not exists"
}
}
backend/locales/zh-CN.json
+41 -39
View File
@@ -1,41 +1,43 @@
{
"i18n_default_workspace": "默认工作空间",
"i18n_ds_name_exist": "名称已存在",
"i18n_concat_admin": "请联系管理员！",
"i18n_exist": "{msg}已存在！",
"i18n_not_exist": "{msg}不存在！",
"i18n_error": "{key}错误！",
"i18n_miss_args": "缺失{key}参数！",
"i18n_format_invalid": "{key}格式不正确！",
"i18n_login": {
"account_pwd_error": "账号或密码错误！",
"no_associated_ws": "没有关联的工作空间，{msg}",
"user_disable": "账号已禁用，{msg}"
},
"i18n_user": {
"account": "账号",
"email": "邮箱",
"password": "密码",
"language_not_support": "系统不支持[{key}]语言！",
"ws_miss": "当前用户不在工作空间[{ws}]中！"
},
"i18n_ws": {
"title": "工作空间"
},
"i18n_permission": {
"only_admin": "仅支持管理员调用！",
"no_permission": "无权调用{url}{msg}",
"authenticate_invalid": "认证无效【{msg}】",
"token_expired": "Token 已过期"
},
"i18n_llm": {
"validate_error": "校验失败[{msg}]",
"delete_default_error": "无法删除默认模型[{key}]！",
"miss_default": "尚未配置默认大语言模型"
},
"i18n_ds_invalid": "数据源连接无效",
"i18n_embedded": {
"invalid_origin": "域名校验失败【{origin}】"
}
"i18n_default_workspace": "默认工作空间",
"i18n_ds_name_exist": "名称已存在",
"i18n_concat_admin": "请联系管理员！",
"i18n_exist": "{msg}已存在！",
"i18n_not_exist": "{msg}不存在！",
"i18n_error": "{key}错误！",
"i18n_miss_args": "缺失{key}参数！",
"i18n_format_invalid": "{key}格式不正确！",
"i18n_login": {
"account_pwd_error": "账号或密码错误！",
"no_associated_ws": "没有关联的工作空间，{msg}",
"user_disable": "账号已禁用，{msg}"
},
"i18n_user": {
"account": "账号",
"email": "邮箱",
"password": "密码",
"language_not_support": "系统不支持[{key}]语言！",
"ws_miss": "当前用户不在工作空间[{ws}]中！"
},
"i18n_ws": {
"title": "工作空间"
},
"i18n_permission": {
"only_admin": "仅支持管理员调用！",
"no_permission": "无权调用{url}{msg}",
"authenticate_invalid": "认证无效【{msg}】",
"token_expired": "Token 已过期"
},
"i18n_llm": {
"validate_error": "校验失败[{msg}]",
"delete_default_error": "无法删除默认模型[{key}]！",
"miss_default": "尚未配置默认大语言模型"
},
"i18n_ds_invalid": "数据源连接无效",
"i18n_embedded": {
"invalid_origin": "域名校验失败【{origin}】"
},
"i18n_terminology": {
"terminology_not_exists": "该术语不存在"
}
}
backend/template.yaml
+23 -3
View File
@@ -12,7 +12,7 @@ template:
<Infos>内有<db-engine><m-schema><terminologies>等信息；
其中，<db-engine>：提供数据库引擎及版本信息；
<m-schema>：以 M-Schema 格式提供数据库表结构信息；
<terminologies>：提供一组专业术语，块内每一个<terminology>就是专业术语，其中同一个<words>内的多个<word>代表术语的多种叫法，也就是术语与它的同义词，<description>即该专业术语对应的专业术语的描述，其中也可能是能够用来参考的计算公式
<terminologies>：提供一组专业术语，块内每一个<terminology>就是专业术语，其中同一个<words>内的多个<word>代表术语的多种叫法，也就是术语与它的同义词，<description>即该专业术语对应的专业术语的描述，其中也可能是能够用来参考的计算公式，或者是一些其他的查询条件
用户的提问在<user-question>内，<error-msg>内则会提供上次执行你提供的SQL时会出现的错误信息，<background-infos>内的<current-time>会告诉你用户当前提问的时间
</Instruction>
@@ -128,6 +128,13 @@ template:
</words>
<description>指在一个季度或一年，一个国家或地区的经济中所生产出的全部最终产品和劳务的价值。</description>
</terminology>
<terminology>
<words>
<word>中国</word>
<word>中国大陆</word>
</words>
<description>查询SQL时若作为查询条件，将"中国"作为查询用的值</description>
</terminology>
</terminologies>
</Info>
@@ -166,7 +173,7 @@ template:
<user-question>查询各个国家每年的GDP</user-question>
</input>
<output>
{{"success":true,"sql":"SELECT \"country\" AS \"country_alias\", \"continent\" AS \"continent_alias\", \"year\" AS \"year_alias\", \"gdp\" AS \"gdp_alias\" FROM \"Sample_Database\".\"sample_country_gdp\" ORDER BY \"country\", \"year\" LIMIT 1000","tables":["sample_country_gdp"],"chart-type":"line"}}
{{"success":true,"sql":"SELECT \"country\" AS \"country_name\", \"continent\" AS \"continent_name\", \"year\" AS \"year\", \"gdp\" AS \"gdp\" FROM \"Sample_Database\".\"sample_country_gdp\" ORDER BY \"country\", \"year\" LIMIT 1000","tables":["sample_country_gdp"],"chart-type":"line"}}
</output>
</example>
<example>
@@ -178,7 +185,20 @@ template:
<background-infos>
<user-question>使用饼图展示去年各个国家的GDP</user-question>
</input>
{{"success":true,"sql":"SELECT \"country\" AS \"country_alias\", \"gdp\" AS \"gdp_alias\" FROM \"Sample_Database\".\"sample_country_gdp\" WHERE \"year\" = '2024' ORDER BY \"gdp\" DESC LIMIT 1000","tables":["sample_country_gdp"],"chart-type":"pie"}}
{{"success":true,"sql":"SELECT \"country\" AS \"country_name\", \"gdp\" AS \"gdp\" FROM \"Sample_Database\".\"sample_country_gdp\" WHERE \"year\" = '2024' ORDER BY \"gdp\" DESC LIMIT 1000","tables":["sample_country_gdp"],"chart-type":"pie"}}
<output>
</output>
</example>
<example>
<input>
<background-infos>
<current-time>
2025-08-08 11:24:00
</current-time>
<background-infos>
<user-question>查询今年中国大陆的GDP</user-question>
</input>
{{"success":true,"sql":"SELECT \"country\" AS \"country_name\", \"gdp\" AS \"gdp\" FROM \"Sample_Database\".\"sample_country_gdp\" WHERE \"year\" = '2025' AND \"国家\" = '中国' LIMIT 1000","tables":["sample_country_gdp"],"chart-type":"table"}}
<output>
</output>
</example>