From 00b9813f7f2867d3f349f58c5d4cb4473a57f896 Mon Sep 17 00:00:00 2001 From: ulleo Date: Tue, 11 Nov 2025 11:48:48 +0800 Subject: [PATCH] feat: Advanced Application support to use SQL Examples --- backend/apps/chat/task/llm.py | 16 ++- .../apps/data_training/curd/data_training.py | 109 ++++++++++++----- .../models/data_training_model.py | 16 +++ backend/apps/system/api/assistant.py | 86 +++++++------ backend/locales/en.json | 3 +- backend/locales/ko-KR.json | 1 + backend/locales/zh-CN.json | 1 + frontend/src/api/embedded.ts | 2 + frontend/src/views/system/training/index.vue | 113 +++++++++++++----- 9 files changed, 251 insertions(+), 96 deletions(-) diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index cd4c35b3..7d81d2c8 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -505,8 +505,12 @@ class LLMService: self.chat_question.terminologies = get_terminology_template(_session, self.chat_question.question, oid, ds_id) - self.chat_question.data_training = get_training_template(_session, self.chat_question.question, ds_id, - oid) + if self.current_assistant and self.current_assistant.type == 1: + self.chat_question.data_training = get_training_template(_session, self.chat_question.question, + oid, None, self.current_assistant.id) + else: + self.chat_question.data_training = get_training_template(_session, self.chat_question.question, + oid, ds_id) if SQLBotLicenseUtil.valid(): self.chat_question.custom_prompt = find_custom_prompts(_session, CustomPromptTypeEnum.GENERATE_SQL, oid, ds_id) @@ -902,8 +906,12 @@ class LLMService: ds_id = self.ds.id if isinstance(self.ds, CoreDatasource) else None self.chat_question.terminologies = get_terminology_template(_session, self.chat_question.question, oid, ds_id) - self.chat_question.data_training = get_training_template(_session, self.chat_question.question, - ds_id, oid) + if self.current_assistant and self.current_assistant.type == 1: + self.chat_question.data_training = get_training_template(_session, self.chat_question.question, + oid, None, self.current_assistant.id) + else: + self.chat_question.data_training = get_training_template(_session, self.chat_question.question, + oid, ds_id) if SQLBotLicenseUtil.valid(): self.chat_question.custom_prompt = find_custom_prompts(_session, CustomPromptTypeEnum.GENERATE_SQL, diff --git a/backend/apps/data_training/curd/data_training.py b/backend/apps/data_training/curd/data_training.py index 7bc51b15..12b99727 100644 --- a/backend/apps/data_training/curd/data_training.py +++ b/backend/apps/data_training/curd/data_training.py @@ -9,8 +9,9 @@ from sqlalchemy import and_, select, func, delete, update, or_ from sqlalchemy import text from apps.ai_model.embedding import EmbeddingModelCache -from apps.data_training.models.data_training_model import DataTrainingInfo, DataTraining +from apps.data_training.models.data_training_model import DataTrainingInfo, DataTraining, DataTrainingInfoResult from apps.datasource.models.datasource import CoreDatasource +from apps.system.models.system_model import AssistantModel from apps.template.generate_chart.generator import get_base_data_training_template from common.core.config import settings from common.core.deps import SessionDep, Trans @@ -19,7 +20,7 @@ from common.utils.embedding_threads import run_save_data_training_embeddings def page_data_training(session: SessionDep, current_page: int = 1, page_size: int = 10, name: Optional[str] = None, oid: Optional[int] = 1): - _list: List[DataTrainingInfo] = [] + _list: List[DataTrainingInfoResult] = [] current_page = max(1, current_page) page_size = max(10, page_size) @@ -63,8 +64,12 @@ def page_data_training(session: SessionDep, current_page: int = 1, page_size: in DataTraining.create_time, DataTraining.description, DataTraining.enabled, + DataTraining.advanced_application, + AssistantModel.name.label('advanced_application_name'), ) .outerjoin(CoreDatasource, and_(DataTraining.datasource == CoreDatasource.id)) + .outerjoin(AssistantModel, + and_(DataTraining.advanced_application == AssistantModel.id, AssistantModel.type == 1)) .where(and_(DataTraining.id.in_(paginated_parent_ids))) .order_by(DataTraining.create_time.desc()) ) @@ -72,15 +77,17 @@ def page_data_training(session: SessionDep, current_page: int = 1, page_size: in result = session.execute(stmt) for row in result: - _list.append(DataTrainingInfo( - id=row.id, - oid=row.oid, + _list.append(DataTrainingInfoResult( + id=str(row.id), + oid=str(row.oid), datasource=row.datasource, datasource_name=row.name, question=row.question, create_time=row.create_time, description=row.description, enabled=row.enabled, + advanced_application=str(row.advanced_application) if row.advanced_application else None, + advanced_application_name=row.advanced_application_name, )) return current_page, page_size, total_count, total_pages, _list @@ -88,15 +95,29 @@ def page_data_training(session: SessionDep, current_page: int = 1, page_size: in def create_training(session: SessionDep, info: DataTrainingInfo, oid: int, trans: Trans): create_time = datetime.datetime.now() - if info.datasource is None: - raise Exception(trans("i18n_data_training.datasource_cannot_be_none")) + if info.datasource is None and info.advanced_application is None: + if oid == 1: + raise Exception(trans("i18n_data_training.datasource_assistant_cannot_be_none")) + else: + raise Exception(trans("i18n_data_training.datasource_cannot_be_none")) + parent = DataTraining(question=info.question, create_time=create_time, description=info.description, oid=oid, - datasource=info.datasource, enabled=info.enabled) + datasource=info.datasource, enabled=info.enabled, + advanced_application=info.advanced_application) + + stmt = select(DataTraining.id).where(and_(DataTraining.question == info.question, DataTraining.oid == oid)) + + if info.datasource is not None and info.advanced_application is not None: + stmt = stmt.where( + or_(DataTraining.datasource == info.datasource, + DataTraining.advanced_application == info.advanced_application)) + elif info.datasource is not None and info.advanced_application is None: + stmt = stmt.where(and_(DataTraining.datasource == info.datasource)) + elif info.datasource is None and info.advanced_application is not None: + stmt = stmt.where(and_(DataTraining.advanced_application == info.advanced_application)) + + exists = session.query(stmt.exists()).scalar() - exists = session.query( - session.query(DataTraining).filter( - and_(DataTraining.question == info.question, DataTraining.oid == oid, - DataTraining.datasource == info.datasource)).exists()).scalar() if exists: raise Exception(trans("i18n_data_training.exists_in_db")) @@ -116,8 +137,11 @@ def create_training(session: SessionDep, info: DataTrainingInfo, oid: int, trans def update_training(session: SessionDep, info: DataTrainingInfo, oid: int, trans: Trans): - if info.datasource is None: - raise Exception(trans("i18n_data_training.datasource_cannot_be_none")) + if info.datasource is None and info.advanced_application is None: + if oid == 1: + raise Exception(trans("i18n_data_training.datasource_assistant_cannot_be_none")) + else: + raise Exception(trans("i18n_data_training.datasource_cannot_be_none")) count = session.query(DataTraining).filter( DataTraining.id == info.id @@ -125,11 +149,20 @@ def update_training(session: SessionDep, info: DataTrainingInfo, oid: int, trans if count == 0: raise Exception(trans('i18n_data_training.data_training_not_exists')) - exists = session.query( - session.query(DataTraining).filter( - and_(DataTraining.question == info.question, DataTraining.oid == oid, - DataTraining.datasource == info.datasource, - DataTraining.id != info.id)).exists()).scalar() + stmt = select(DataTraining.id).where( + and_(DataTraining.question == info.question, DataTraining.oid == oid, DataTraining.id != info.id)) + + if info.datasource is not None and info.advanced_application is not None: + stmt = stmt.where( + or_(DataTraining.datasource == info.datasource, + DataTraining.advanced_application == info.advanced_application)) + elif info.datasource is not None and info.advanced_application is None: + stmt = stmt.where(and_(DataTraining.datasource == info.datasource)) + elif info.datasource is None and info.advanced_application is not None: + stmt = stmt.where(and_(DataTraining.advanced_application == info.advanced_application)) + + exists = session.query(stmt.exists()).scalar() + if exists: raise Exception(trans("i18n_data_training.exists_in_db")) @@ -138,6 +171,7 @@ def update_training(session: SessionDep, info: DataTrainingInfo, oid: int, trans description=info.description, datasource=info.datasource, enabled=info.enabled, + advanced_application=info.advanced_application, ) session.execute(stmt) session.commit() @@ -231,9 +265,21 @@ WHERE similarity > {settings.EMBEDDING_DATA_TRAINING_SIMILARITY} and oid = :oid ORDER BY similarity DESC LIMIT {settings.EMBEDDING_DATA_TRAINING_TOP_COUNT} """ +embedding_sql_in_advanced_application = f""" +SELECT id, datasource, question, similarity +FROM +(SELECT id, datasource, question, oid, enabled, +( 1 - (embedding <=> :embedding_array) ) AS similarity +FROM data_training AS child +) TEMP +WHERE similarity > {settings.EMBEDDING_DATA_TRAINING_SIMILARITY} and oid = :oid and advanced_application = :advanced_application and enabled = true +ORDER BY similarity DESC +LIMIT {settings.EMBEDDING_DATA_TRAINING_TOP_COUNT} +""" -def select_training_by_question(session: SessionDep, question: str, oid: int, datasource: int): +def select_training_by_question(session: SessionDep, question: str, oid: int, datasource: Optional[int] = None, + advanced_application_id: Optional[int] = None): if question.strip() == "": return [] @@ -248,10 +294,13 @@ def select_training_by_question(session: SessionDep, question: str, oid: int, da .where( and_(or_(text(":sentence ILIKE '%' || question || '%'"), text("question ILIKE '%' || :sentence || '%'")), DataTraining.oid == oid, - DataTraining.datasource == datasource, - DataTraining.enabled == True,) + DataTraining.enabled == True) ) ) + if advanced_application_id is not None: + stmt = stmt.where(and_(DataTraining.advanced_application == advanced_application_id)) + else: + stmt = stmt.where(and_(DataTraining.datasource == datasource)) results = session.execute(stmt, {'sentence': question}).fetchall() @@ -264,8 +313,13 @@ def select_training_by_question(session: SessionDep, question: str, oid: int, da embedding = model.embed_query(question) - results = session.execute(text(embedding_sql), - {'embedding_array': str(embedding), 'oid': oid, 'datasource': datasource}) + if advanced_application_id is not None: + results = session.execute(text(embedding_sql_in_advanced_application), + {'embedding_array': str(embedding), 'oid': oid, + 'advanced_application': advanced_application_id}) + else: + results = session.execute(text(embedding_sql), + {'embedding_array': str(embedding), 'oid': oid, 'datasource': datasource}) for row in results: _list.append(DataTraining(id=row.id, question=row.question)) @@ -328,12 +382,13 @@ def to_xml_string(_dict: list[dict] | dict, root: str = 'sql-examples') -> str: return pretty_xml -def get_training_template(session: SessionDep, question: str, datasource: int, oid: Optional[int] = 1) -> str: +def get_training_template(session: SessionDep, question: str, oid: Optional[int] = 1, datasource: Optional[int] = None, + advanced_application_id: Optional[int] = None) -> str: if not oid: oid = 1 - if not datasource: + if not datasource and not advanced_application_id: return '' - _results = select_training_by_question(session, question, oid, datasource) + _results = select_training_by_question(session, question, oid, datasource, advanced_application_id) if _results and len(_results) > 0: data_training = to_xml_string(_results) template = get_base_data_training_template().format(data_training=data_training) diff --git a/backend/apps/data_training/models/data_training_model.py b/backend/apps/data_training/models/data_training_model.py index 24e69be9..45e8c3a8 100644 --- a/backend/apps/data_training/models/data_training_model.py +++ b/backend/apps/data_training/models/data_training_model.py @@ -17,6 +17,7 @@ class DataTraining(SQLModel, table=True): description: Optional[str] = Field(sa_column=Column(Text, nullable=True)) embedding: Optional[List[float]] = Field(sa_column=Column(VECTOR(), nullable=True)) enabled: Optional[bool] = Field(sa_column=Column(Boolean, default=True)) + advanced_application: Optional[int] = Field(sa_column=Column(BigInteger, nullable=True)) class DataTrainingInfo(BaseModel): @@ -28,3 +29,18 @@ class DataTrainingInfo(BaseModel): question: Optional[str] = None description: Optional[str] = None enabled: Optional[bool] = True + advanced_application: Optional[int] = None + advanced_application_name: Optional[str] = None + + +class DataTrainingInfoResult(BaseModel): + id: Optional[str] = None + oid: Optional[str] = None + datasource: Optional[int] = None + datasource_name: Optional[str] = None + create_time: Optional[datetime] = None + question: Optional[str] = None + description: Optional[str] = None + enabled: Optional[bool] = True + advanced_application: Optional[str] = None + advanced_application_name: Optional[str] = None diff --git a/backend/apps/system/api/assistant.py b/backend/apps/system/api/assistant.py index 69b1cc0b..9733d3f0 100644 --- a/backend/apps/system/api/assistant.py +++ b/backend/apps/system/api/assistant.py @@ -1,27 +1,28 @@ -from datetime import timedelta import json import os +from datetime import timedelta from typing import List, Optional + from fastapi import APIRouter, Form, HTTPException, Query, Request, Response, UploadFile from fastapi.responses import StreamingResponse +from sqlbot_xpack.file_utils import SQLBotFileUtils from sqlmodel import select + from apps.system.crud.assistant import get_assistant_info from apps.system.crud.assistant_manage import dynamic_upgrade_cors, save from apps.system.models.system_model import AssistantModel from apps.system.schemas.auth import CacheName, CacheNamespace from apps.system.schemas.system_schema import AssistantBase, AssistantDTO, AssistantUiSchema, AssistantValidator +from common.core.config import settings from common.core.deps import SessionDep, Trans from common.core.security import create_access_token from common.core.sqlbot_cache import clear_cache -from common.utils.time import get_timestamp - -from common.core.config import settings from common.utils.utils import get_origin_from_referer -from sqlbot_xpack.file_utils import SQLBotFileUtils router = APIRouter(tags=["system/assistant"], prefix="/system/assistant") -@router.get("/info/{id}") + +@router.get("/info/{id}") async def info(request: Request, response: Response, session: SessionDep, trans: Trans, id: int) -> AssistantModel: if not id: raise Exception('miss assistant id') @@ -32,13 +33,14 @@ async def info(request: Request, response: Response, session: SessionDep, trans: response.headers["Access-Control-Allow-Origin"] = db_model.domain origin = request.headers.get("origin") or get_origin_from_referer(request) if not origin: - raise RuntimeError(trans('i18n_embedded.invalid_origin', origin = origin or '')) + raise RuntimeError(trans('i18n_embedded.invalid_origin', origin=origin or '')) origin = origin.rstrip('/') if origin != db_model.domain: - raise RuntimeError(trans('i18n_embedded.invalid_origin', origin = origin or '')) + raise RuntimeError(trans('i18n_embedded.invalid_origin', origin=origin or '')) return db_model -@router.get("/app/{appId}") + +@router.get("/app/{appId}") async def getApp(request: Request, response: Response, session: SessionDep, trans: Trans, appId: str) -> AssistantModel: if not appId: raise Exception('miss assistant appId') @@ -49,27 +51,28 @@ async def getApp(request: Request, response: Response, session: SessionDep, tran response.headers["Access-Control-Allow-Origin"] = db_model.domain origin = request.headers.get("origin") or get_origin_from_referer(request) if not origin: - raise RuntimeError(trans('i18n_embedded.invalid_origin', origin = origin or '')) + raise RuntimeError(trans('i18n_embedded.invalid_origin', origin=origin or '')) origin = origin.rstrip('/') if origin != db_model.domain: - raise RuntimeError(trans('i18n_embedded.invalid_origin', origin = origin or '')) + raise RuntimeError(trans('i18n_embedded.invalid_origin', origin=origin or '')) return db_model -@router.get("/validator", response_model=AssistantValidator) + +@router.get("/validator", response_model=AssistantValidator) async def validator(session: SessionDep, id: int, virtual: Optional[int] = Query(None)): if not id: raise Exception('miss assistant id') - + db_model = await get_assistant_info(session=session, assistant_id=id) if not db_model: return AssistantValidator() db_model = AssistantModel.model_validate(db_model) assistant_oid = 1 - if(db_model.type == 0): + if (db_model.type == 0): configuration = db_model.configuration config_obj = json.loads(configuration) if configuration else {} assistant_oid = config_obj.get('oid', 1) - + access_token_expires = timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES) assistantDict = { "id": virtual, "account": 'sqlbot-inner-assistant', "oid": assistant_oid, "assistant_id": id @@ -78,24 +81,26 @@ async def validator(session: SessionDep, id: int, virtual: Optional[int] = Query assistantDict, expires_delta=access_token_expires ) return AssistantValidator(True, True, True, access_token) - + + @router.get('/picture/{file_id}') async def picture(file_id: str): file_path = SQLBotFileUtils.get_file_path(file_id=file_id) if not os.path.exists(file_path): raise HTTPException(status_code=404, detail="File not found") - + if file_id.lower().endswith(".svg"): media_type = "image/svg+xml" else: media_type = "image/jpeg" - + def iterfile(): with open(file_path, mode="rb") as f: yield from f - + return StreamingResponse(iterfile(), media_type=media_type) + @router.patch('/ui') async def ui(session: SessionDep, data: str = Form(), files: List[UploadFile] = []): json_data = json.loads(data) @@ -106,7 +111,7 @@ async def ui(session: SessionDep, data: str = Form(), files: List[UploadFile] = raise ValueError(f"AssistantModel with id {id} not found") configuration = db_model.configuration config_obj = json.loads(configuration) if configuration else {} - + ui_schema_dict = uiSchema.model_dump(exclude_none=True, exclude_unset=True) if files: for file in files: @@ -114,45 +119,57 @@ async def ui(session: SessionDep, data: str = Form(), files: List[UploadFile] = file_name, flag_name = SQLBotFileUtils.split_filename_and_flag(origin_file_name) file.filename = file_name if flag_name == 'logo' or flag_name == 'float_icon': - SQLBotFileUtils.check_file(file=file, file_types=[".jpg", ".jpeg", ".png", ".svg"], limit_file_size=(10 * 1024 * 1024)) + SQLBotFileUtils.check_file(file=file, file_types=[".jpg", ".jpeg", ".png", ".svg"], + limit_file_size=(10 * 1024 * 1024)) if config_obj.get(flag_name): SQLBotFileUtils.delete_file(config_obj.get(flag_name)) file_id = await SQLBotFileUtils.upload(file) ui_schema_dict[flag_name] = file_id else: raise ValueError(f"Unsupported file flag: {flag_name}") - + for flag_name in ['logo', 'float_icon']: file_val = config_obj.get(flag_name) if file_val and not ui_schema_dict.get(flag_name): config_obj[flag_name] = None SQLBotFileUtils.delete_file(file_val) - + for attr, value in ui_schema_dict.items(): if attr != 'id' and not attr.startswith("__"): config_obj[attr] = value - + db_model.configuration = json.dumps(config_obj, ensure_ascii=False) session.add(db_model) session.commit() await clear_ui_cache(db_model.id) -@clear_cache(namespace=CacheNamespace.EMBEDDED_INFO, cacheName=CacheName.ASSISTANT_INFO, keyExpression="id") + +@clear_cache(namespace=CacheNamespace.EMBEDDED_INFO, cacheName=CacheName.ASSISTANT_INFO, keyExpression="id") async def clear_ui_cache(id: int): pass + @router.get("", response_model=list[AssistantModel]) async def query(session: SessionDep): - list_result = session.exec(select(AssistantModel).where(AssistantModel.type != 4).order_by(AssistantModel.name, AssistantModel.create_time)).all() + list_result = session.exec(select(AssistantModel).where(AssistantModel.type != 4).order_by(AssistantModel.name, + AssistantModel.create_time)).all() return list_result + +@router.get("/advanced_application", response_model=list[AssistantModel]) +async def query_advanced_application(session: SessionDep): + list_result = session.exec(select(AssistantModel).where(AssistantModel.type == 1).order_by(AssistantModel.name, + AssistantModel.create_time)).all() + return list_result + + @router.post("") async def add(request: Request, session: SessionDep, creator: AssistantBase): await save(request, session, creator) - + @router.put("") -@clear_cache(namespace=CacheNamespace.EMBEDDED_INFO, cacheName=CacheName.ASSISTANT_INFO, keyExpression="editor.id") +@clear_cache(namespace=CacheNamespace.EMBEDDED_INFO, cacheName=CacheName.ASSISTANT_INFO, keyExpression="editor.id") async def update(request: Request, session: SessionDep, editor: AssistantDTO): id = editor.id db_model = session.get(AssistantModel, id) @@ -164,7 +181,8 @@ async def update(request: Request, session: SessionDep, editor: AssistantDTO): session.commit() dynamic_upgrade_cors(request=request, session=session) -@router.get("/{id}", response_model=AssistantModel) + +@router.get("/{id}", response_model=AssistantModel) async def get_one(session: SessionDep, id: int): db_model = await get_assistant_info(session=session, assistant_id=id) if not db_model: @@ -172,8 +190,9 @@ async def get_one(session: SessionDep, id: int): db_model = AssistantModel.model_validate(db_model) return db_model + @router.delete("/{id}") -@clear_cache(namespace=CacheNamespace.EMBEDDED_INFO, cacheName=CacheName.ASSISTANT_INFO, keyExpression="id") +@clear_cache(namespace=CacheNamespace.EMBEDDED_INFO, cacheName=CacheName.ASSISTANT_INFO, keyExpression="id") async def delete(request: Request, session: SessionDep, id: int): db_model = session.get(AssistantModel, id) if not db_model: @@ -181,10 +200,3 @@ async def delete(request: Request, session: SessionDep, id: int): session.delete(db_model) session.commit() dynamic_upgrade_cors(request=request, session=session) - - - - - - - diff --git a/backend/locales/en.json b/backend/locales/en.json index 049c47aa..6d922a4c 100644 --- a/backend/locales/en.json +++ b/backend/locales/en.json @@ -46,10 +46,11 @@ }, "i18n_data_training": { "datasource_cannot_be_none": "Datasource cannot be none or empty", + "datasource_assistant_cannot_be_none": "Datasource or Advanced application cannot be all none", "data_training_not_exists": "Example does not exists", "exists_in_db": "Question exists" }, - "i18n_custom_prompt": { + "i18n_custom_prompt": { "exists_in_db": "Prompt name exists", "not_exists": "Prompt does not exists" }, diff --git a/backend/locales/ko-KR.json b/backend/locales/ko-KR.json index b426ecc3..03528a31 100644 --- a/backend/locales/ko-KR.json +++ b/backend/locales/ko-KR.json @@ -46,6 +46,7 @@ }, "i18n_data_training": { "datasource_cannot_be_none": "데이터 소스를 선택해 주세요", + "datasource_assistant_cannot_be_none": "데이터 소스와 고급 애플리케이션이 모두 비어 있을 수는 없습니다.", "data_training_not_exists": "예제가 존재하지 않습니다", "exists_in_db": "질문이 이미 존재합니다" }, diff --git a/backend/locales/zh-CN.json b/backend/locales/zh-CN.json index 010943d0..7a593a09 100644 --- a/backend/locales/zh-CN.json +++ b/backend/locales/zh-CN.json @@ -46,6 +46,7 @@ }, "i18n_data_training": { "datasource_cannot_be_none": "数据源不能为空", + "datasource_assistant_cannot_be_none": "数据源或高级应用不能都为空", "data_training_not_exists": "该示例不存在", "exists_in_db": "该问题已存在" }, diff --git a/frontend/src/api/embedded.ts b/frontend/src/api/embedded.ts index 5c70c721..0733088a 100644 --- a/frontend/src/api/embedded.ts +++ b/frontend/src/api/embedded.ts @@ -1,6 +1,8 @@ import { request } from '@/utils/request' export const getList = () => request.get('/system/assistant') +export const getAdvancedApplicationList = () => + request.get('/system/assistant/advanced_application') export const updateAssistant = (data: any) => request.put('/system/assistant', data) export const saveAssistant = (data: any) => request.post('/system/assistant', data) export const getOne = (id: any) => request.get(`/system/assistant/${id}`) diff --git a/frontend/src/views/system/training/index.vue b/frontend/src/views/system/training/index.vue index 97a41cf3..9038b4c9 100644 --- a/frontend/src/views/system/training/index.vue +++ b/frontend/src/views/system/training/index.vue @@ -1,5 +1,5 @@