From 7cd4d783cdb1accbcd3dc0dc1d76317a9b663ae6 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Tue, 29 Jul 2025 09:34:37 +0800 Subject: [PATCH 1/9] perf: Assistant dynamic ds api param --- backend/apps/system/crud/assistant.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/apps/system/crud/assistant.py b/backend/apps/system/crud/assistant.py index c916262b..9b1579fc 100644 --- a/backend/apps/system/crud/assistant.py +++ b/backend/apps/system/crud/assistant.py @@ -101,13 +101,15 @@ class AssistantOutDs: certificateList: list[any] = json.loads(self.certificate) header = {} cookies = {} + param = {} for item in certificateList: if item['target'] == 'header': header[item['key']] = item['value'] if item['target'] == 'cookie': cookies[item['key']] = item['value'] - - res = requests.get(url=endpoint, headers=header, cookies=cookies, timeout=10) + if item['target'] == 'param': + param[item['key']] = item['value'] + res = requests.get(url=endpoint, params=param, headers=header, cookies=cookies, timeout=10) if res.status_code == 200: result_json: dict[any] = json.loads(res.text) if result_json.get('code') == 0: From e575551236361415c67d0fa25b4ec07f458ad456 Mon Sep 17 00:00:00 2001 From: ulleo Date: Tue, 29 Jul 2025 10:17:51 +0800 Subject: [PATCH 2/9] fix: chat generate sql error --- backend/apps/chat/models/chat_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/apps/chat/models/chat_model.py b/backend/apps/chat/models/chat_model.py index 7aba311d..e86fc11f 100644 --- a/backend/apps/chat/models/chat_model.py +++ b/backend/apps/chat/models/chat_model.py @@ -147,7 +147,7 @@ class AiModelQuestion(BaseModel): old_questions=old_questions) def filter_sys_question(self): - return get_permissions_template()['system'].format(lang=self.lang) + return get_permissions_template()['system'].format(lang=self.lang, engine=self.engine) def filter_user_question(self): return get_permissions_template()['user'].format(sql=self.sql, filter=self.filter) From 2efad70b8dcd0b91405ef25395e8481f9a168016 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Tue, 29 Jul 2025 10:25:54 +0800 Subject: [PATCH 3/9] perf: Ignore select ds if only one ds --- backend/apps/chat/task/llm.py | 100 ++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index 624d4bed..cbf48e2a 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -386,51 +386,56 @@ class LLMService: ] """ _ds_list = self.session.exec(select(CoreDatasource).options( load_only(CoreDatasource.id, CoreDatasource.name, CoreDatasource.description))).all() """ - _ds_list_dict = [] - for _ds in _ds_list: - _ds_list_dict.append(_ds) - datasource_msg.append( - HumanMessage(self.chat_question.datasource_user_question(orjson.dumps(_ds_list_dict).decode()))) + + ignore_auto_select = _ds_list and len(_ds_list) == 1 + # ignore auto select ds + + if not ignore_auto_select: + _ds_list_dict = [] + for _ds in _ds_list: + _ds_list_dict.append(_ds) + datasource_msg.append( + HumanMessage(self.chat_question.datasource_user_question(orjson.dumps(_ds_list_dict).decode()))) - history_msg = [] - if self.record.full_select_datasource_message and self.record.full_select_datasource_message.strip() != '': - history_msg = orjson.loads(self.record.full_select_datasource_message) + history_msg = [] + if self.record.full_select_datasource_message and self.record.full_select_datasource_message.strip() != '': + history_msg = orjson.loads(self.record.full_select_datasource_message) - self.record = save_full_select_datasource_message_and_answer(session=self.session, record_id=self.record.id, - answer='', - full_message=orjson.dumps(history_msg + - [{'type': msg.type, - 'content': msg.content} - for msg - in - datasource_msg]).decode()) - full_thinking_text = '' - full_text = '' - token_usage = {} - res = self.llm.stream(datasource_msg) - for chunk in res: - SQLBotLogUtil.info(chunk) - reasoning_content_chunk = '' - if 'reasoning_content' in chunk.additional_kwargs: - reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '') - # else: - # reasoning_content_chunk = chunk.get('reasoning_content') - if reasoning_content_chunk is None: + self.record = save_full_select_datasource_message_and_answer(session=self.session, record_id=self.record.id, + answer='', + full_message=orjson.dumps(history_msg + + [{'type': msg.type, + 'content': msg.content} + for msg + in + datasource_msg]).decode()) + full_thinking_text = '' + full_text = '' + token_usage = {} + res = self.llm.stream(datasource_msg) + for chunk in res: + SQLBotLogUtil.info(chunk) reasoning_content_chunk = '' - full_thinking_text += reasoning_content_chunk + if 'reasoning_content' in chunk.additional_kwargs: + reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '') + # else: + # reasoning_content_chunk = chunk.get('reasoning_content') + if reasoning_content_chunk is None: + reasoning_content_chunk = '' + full_thinking_text += reasoning_content_chunk - full_text += chunk.content - yield {'content': chunk.content, 'reasoning_content': reasoning_content_chunk} - get_token_usage(chunk, token_usage) - datasource_msg.append(AIMessage(full_text)) + full_text += chunk.content + yield {'content': chunk.content, 'reasoning_content': reasoning_content_chunk} + get_token_usage(chunk, token_usage) + datasource_msg.append(AIMessage(full_text)) - json_str = extract_nested_json(full_text) + json_str = extract_nested_json(full_text) _error: Exception | None = None _datasource: int | None = None _engine_type: str | None = None try: - data: dict = orjson.loads(json_str) + data: dict = _ds_list[0] if ignore_auto_select else orjson.loads(json_str) if data.get('id') and data.get('id') != 0: _datasource = data['id'] @@ -465,17 +470,18 @@ class LLMService: except Exception as e: _error = e - self.record = save_full_select_datasource_message_and_answer(session=self.session, record_id=self.record.id, - answer=orjson.dumps({'content': full_text, - 'reasoning_content': full_thinking_text}).decode(), - datasource=_datasource, - engine_type=_engine_type, - full_message=orjson.dumps(history_msg + - [{'type': msg.type, - 'content': msg.content} - for msg - in - datasource_msg]).decode()) + if not ignore_auto_select: + self.record = save_full_select_datasource_message_and_answer(session=self.session, record_id=self.record.id, + answer=orjson.dumps({'content': full_text, + 'reasoning_content': full_thinking_text}).decode(), + datasource=_datasource, + engine_type=_engine_type, + full_message=orjson.dumps(history_msg + + [{'type': msg.type, + 'content': msg.content} + for msg + in + datasource_msg]).decode()) self.init_messages() if _error: @@ -815,7 +821,7 @@ class LLMService: SQLBotLogUtil.info(full_sql_text) # todo row permission - if is_normal_user(self.current_user) or (self.current_assistant and self.current_assistant.type == 1): + if (not self.current_assistant and is_normal_user(self.current_user)) or (self.current_assistant and self.current_assistant.type == 1): sql_json_str = extract_nested_json(full_sql_text) data = orjson.loads(sql_json_str) From f689cbc203e3d6542f1701fb8c91e27198d46e8e Mon Sep 17 00:00:00 2001 From: junjun Date: Tue, 29 Jul 2025 11:15:21 +0800 Subject: [PATCH 4/9] feat: timeout --- backend/apps/datasource/crud/datasource.py | 2 +- backend/apps/db/db.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index 52381c6f..a73a22bf 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -41,7 +41,7 @@ def get_ds(session: SessionDep, id: int): def check_status(session: SessionDep, ds: CoreDatasource, is_raise: bool = False): - conn = get_engine(ds) + conn = get_engine(ds, 5) try: with conn.connect() as connection: SQLBotLogUtil.info("success") diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index b051d787..83be9c75 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -52,10 +52,10 @@ def get_uri_from_config(type: str, conf: DatasourceConf) -> str: return db_url -def get_engine(ds: CoreDatasource) -> Engine: +def get_engine(ds: CoreDatasource, timeout: int = 30) -> Engine: conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) if ds.type != "excel" else get_engine_config() if conf.timeout is None: - conf.timeout = 30 + conf.timeout = timeout if ds.type == "pg" and (conf.dbSchema is not None and conf.dbSchema != ""): engine = create_engine(get_uri(ds), connect_args={"options": f"-c search_path={urllib.parse.quote(conf.dbSchema)}", From 257609df5feac2be0081e7e383f2d02caddb5c68 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Tue, 29 Jul 2025 11:15:30 +0800 Subject: [PATCH 5/9] perf: Assistant domain check --- frontend/src/i18n/en.json | 5 +++-- frontend/src/i18n/zh-CN.json | 5 +++-- frontend/src/views/system/embedded/index.vue | 5 +++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index e7645eb2..ddbe077a 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -472,7 +472,8 @@ "duplicate_email": "Duplicate email", "interface_credentials": "Interface credentials", "no_credentials_yet": "No credentials yet", - "intelligent_customer_service": "SQLBot Intelligent Customer Service" + "intelligent_customer_service": "SQLBot Intelligent Customer Service", + "origin_format_error": "Format invalid, starts with http or https and cannot end with /" }, "chat": { "type": "Chart Type", @@ -518,4 +519,4 @@ "back_community": "Restore to Community Edition", "confirm_tips": "Are you sure to restore to Community Edition?" } -} \ No newline at end of file +} diff --git a/frontend/src/i18n/zh-CN.json b/frontend/src/i18n/zh-CN.json index ce571212..8d5014db 100644 --- a/frontend/src/i18n/zh-CN.json +++ b/frontend/src/i18n/zh-CN.json @@ -487,7 +487,8 @@ "stop_replying": "停止回答", "i_am_sqlbot": "你好，我是 SQLBot", "predict_data_etc": "我可以查询数据、生成图表、检测数据异常、预测数据等", - "intelligent_data_query": "赶快开启智能问数吧～" + "intelligent_data_query": "赶快开启智能问数吧～", + "origin_format_error": "格式错误，http或https开头，不能以 / 结尾" }, "chat": { "type": "图表类型", @@ -533,4 +534,4 @@ "back_community": "还原至社区版", "confirm_tips": "确定还原至社区版？" } -} \ No newline at end of file +} diff --git a/frontend/src/views/system/embedded/index.vue b/frontend/src/views/system/embedded/index.vue index 39c4f9b0..cead4863 100644 --- a/frontend/src/views/system/embedded/index.vue +++ b/frontend/src/views/system/embedded/index.vue @@ -245,6 +245,11 @@ const rules = { t('datasource.please_enter') + t('common.empty') + t('embedded.cross_domain_settings'), trigger: 'blur', }, + { + pattern: /^(https?:\/\/)?([\w-]+\.)*[\w-]+(:\d+)?$/, + message: t('embedded.origin_format_error'), + trigger: 'blur', + }, ], } From c068ae5780908d9183682e79ac5a759f2f46251f Mon Sep 17 00:00:00 2001 From: wangjiahao <1522128093@qq.com> Date: Tue, 29 Jul 2025 11:20:06 +0800 Subject: [PATCH 6/9] refactor: support sq add view to dashboard --- .../src/views/chat/chat-block/ChartBlock.vue | 7 +- .../dashboard/common/AddViewDashboard.vue | 249 ++++++++++++++++++ .../dashboard/common/ResourceGroupOpt.vue | 4 - .../dashboard/components/sq-view/index.vue | 1 - .../src/views/dashboard/utils/canvasUtils.ts | 43 ++- 5 files changed, 291 insertions(+), 13 deletions(-) create mode 100644 frontend/src/views/dashboard/common/AddViewDashboard.vue diff --git a/frontend/src/views/chat/chat-block/ChartBlock.vue b/frontend/src/views/chat/chat-block/ChartBlock.vue index 52e59969..96da119a 100644 --- a/frontend/src/views/chat/chat-block/ChartBlock.vue +++ b/frontend/src/views/chat/chat-block/ChartBlock.vue @@ -18,6 +18,7 @@ import icon_window_mini_outlined from '@/assets/svg/icon_window-mini_outlined.sv import icon_copy_outlined from '@/assets/svg/icon_copy_outlined.svg' import { useI18n } from 'vue-i18n' import SQLComponent from '@/views/chat/component/SQLComponent.vue' +import AddViewDashboard from '@/views/dashboard/common/AddViewDashboard.vue' const props = withDefaults( defineProps<{ @@ -34,7 +35,7 @@ const props = withDefaults( ) const { t } = useI18n() - +const addViewRef = ref(null) const emits = defineEmits(['exitFullScreen']) const dataObject = computed<{ @@ -188,7 +189,8 @@ function showSql() { } function addToDashboard() { - console.log('todo') + // @ts-expect-error eslint-disable-next-line @typescript-eslint/ban-ts-comment + addViewRef.value?.optInit() } function copy() { @@ -300,6 +302,7 @@ function copy() { /> + +import { onMounted, reactive, ref } from 'vue' +import { + findNextComponentIndex, + saveDashboardResourceTarget, +} from '@/views/dashboard/utils/canvasUtils.ts' +import { useI18n } from 'vue-i18n' +import { dashboardApi } from '@/api/dashboard.ts' +import type { SQTreeNode } from '@/views/dashboard/utils/treeNode.ts' +import cloneDeep from 'lodash/cloneDeep' +import { findNewComponentFromList } from '@/views/dashboard/components/component-list.ts' +import { guid } from '@/utils/canvas.ts' + +const { t } = useI18n() +const resource = ref(null) + +const optInit = (viewInfo: any) => { + resourceDialogShow.value = true + state.viewInfo = viewInfo +} + +const state = reactive({ + dashboardList: [] as SQTreeNode[], + viewInfo: null, +}) + +const resourceDialogShow = ref(false) +const loading = ref(false) +const resourceForm = reactive({ + addType: 'history', + dashboardId: '', + dashboardName: '', +}) + +const resourceFormRulesNew = ref({ + dashboardName: [ + { + required: true, + min: 1, + max: 64, + message: t('dashboard.length_limit64'), + trigger: 'change', + }, + ], +}) + +const resourceFormRulesHistory = ref({ + dashboardId: [ + { + required: true, + min: 1, + max: 64, + message: '请选择仪表板', + trigger: 'change', + }, + ], +}) + +const resetForm = () => { + resourceForm.dashboardId = '' + resourceForm.dashboardName = '' + resourceDialogShow.value = false +} + +const saveResourcePrepare = () => { + // @ts-expect-error eslint-disable-next-line @typescript-eslint/ban-ts-comment + resource.value?.validate((result) => { + if (result) { + const component = cloneDeep(findNewComponentFromList('SQView')) + const newComponentId = guid() + if (resourceForm.addType === 'history' && component) { + findNextComponentIndex({ id: resourceForm.dashboardId }, (result: any) => { + const { + bottomPosition, + dashboardInfo, + canvasDataResult, + canvasStyleResult, + canvasViewInfoPreview, + } = result + const params = { + opt: 'updateLeaf', + pid: 'root', + id: resourceForm.dashboardId, + name: dashboardInfo.name, + } + component['id'] = newComponentId + component['y'] = bottomPosition + 1 + canvasDataResult.push(component) + canvasViewInfoPreview[newComponentId] = state.viewInfo + const commonParams = { + componentData: canvasDataResult, + canvasStyleData: canvasStyleResult, + canvasViewInfo: canvasViewInfoPreview, + } + saveResource(params, commonParams) + }) + } else if (resourceForm.addType === 'new' && component) { + const params = { opt: 'newLeaf', pid: 'root', name: resourceForm.dashboardName } + component['id'] = newComponentId + const commonParams = { + componentData: [component], + canvasStyleData: {}, + canvasViewInfo: { newComponentId: state.viewInfo }, + } + saveResource(params, commonParams) + } + } + }) +} + +const saveResource = (params: any, commonParams: any) => { + saveDashboardResourceTarget(params, commonParams, function () { + const messageTips = t('common.save_success') + ElMessage({ + type: 'success', + message: messageTips, + }) + resetForm() + }) +} + +const initDashboardList = () => { + state.dashboardList = [] + const params = {} + dashboardApi.list_resource(params).then((res: SQTreeNode[]) => { + state.dashboardList = res || [] + }) +} + +onMounted(() => { + initDashboardList() +}) + +defineExpose({ + optInit, +}) + + + + + diff --git a/frontend/src/views/dashboard/common/ResourceGroupOpt.vue b/frontend/src/views/dashboard/common/ResourceGroupOpt.vue index f552b15e..bc1cdeeb 100644 --- a/frontend/src/views/dashboard/common/ResourceGroupOpt.vue +++ b/frontend/src/views/dashboard/common/ResourceGroupOpt.vue @@ -115,10 +115,6 @@ const propsTree = { isLeaf: (node) => !node.children?.length, } -// const showPid = computed(() => { -// return state.opt && ['newLeaf'].includes(state.opt) && state.parentSelect -// }) - const showPid = false const saveResource = () => { diff --git a/frontend/src/views/dashboard/components/sq-view/index.vue b/frontend/src/views/dashboard/components/sq-view/index.vue index b2a11b49..ca25169b 100644 --- a/frontend/src/views/dashboard/components/sq-view/index.vue +++ b/frontend/src/views/dashboard/components/sq-view/index.vue @@ -2,7 +2,6 @@ import ChartComponent from '@/views/chat/component/ChartComponent.vue' import icon_window_mini_outlined from '@/assets/svg/icon_window-mini_outlined.svg' import SqViewDisplay from '@/views/dashboard/components/sq-view/index.vue' -import _ from 'lodash' const props = defineProps({ viewInfo: { type: Object, diff --git a/frontend/src/views/dashboard/utils/canvasUtils.ts b/frontend/src/views/dashboard/utils/canvasUtils.ts index e7c50f52..99c502df 100644 --- a/frontend/src/views/dashboard/utils/canvasUtils.ts +++ b/frontend/src/views/dashboard/utils/canvasUtils.ts @@ -48,8 +48,39 @@ export const initCanvasData = (params: any, callBack: () => void) => { }) } +// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type +export const findNextComponentIndex = async (params: any, callBack: Function) => { + load_resource_prepare(params, function (result: any) { + let bottomPosition = 0 + const { dashboardInfo, canvasDataResult, canvasStyleResult, canvasViewInfoPreview } = result + canvasDataResult.forEach((component: any) => { + const componentBottom = component.y + component.sizeY + if (componentBottom > bottomPosition) { + bottomPosition = componentBottom + } + }) + callBack({ + bottomPosition, + dashboardInfo, + canvasDataResult, + canvasStyleResult, + canvasViewInfoPreview, + }) + }) +} + // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type export const saveDashboardResource = (params: any, callBack: Function) => { + const commonParams = { + componentData: componentData.value, + canvasStyleData: canvasStyleData.value, + canvasViewInfo: canvasViewInfo.value, + } + saveDashboardResourceTarget(params, commonParams, callBack) +} + +// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type +export const saveDashboardResourceTarget = (params: any, commonParams: any, callBack: Function) => { params['workspace_id'] = workspace_id dashboardApi.check_name(params).then((resCheck: any) => { if (resCheck) { @@ -57,9 +88,9 @@ export const saveDashboardResource = (params: any, callBack: Function) => { // create canvas const requestParams = { ...params, - component_data: JSON.stringify(componentData.value), - canvas_style_data: JSON.stringify(canvasStyleData.value), - canvas_view_info: JSON.stringify(canvasViewInfo.value), + component_data: JSON.stringify(commonParams.componentData), + canvas_style_data: JSON.stringify(commonParams.canvasStyleData), + canvas_view_info: JSON.stringify(commonParams.canvasViewInfo), } dashboardApi.create_canvas(requestParams).then((res: any) => { dashboardStore.updateDashboardInfo({ @@ -81,9 +112,9 @@ export const saveDashboardResource = (params: any, callBack: Function) => { } else if (params.opt === 'updateLeaf') { const requestParams = { ...params, - component_data: JSON.stringify(componentData.value), - canvas_style_data: JSON.stringify(canvasStyleData.value), - canvas_view_info: JSON.stringify(canvasViewInfo.value), + component_data: JSON.stringify(commonParams.componentData), + canvas_style_data: JSON.stringify(commonParams.canvasStyleData), + canvas_view_info: JSON.stringify(commonParams.canvasViewInfo), } dashboardApi.update_canvas(requestParams).then(() => { callBack() From 8141700746aa365cea14c56320b980efededc9c4 Mon Sep 17 00:00:00 2001 From: junjun Date: Tue, 29 Jul 2025 11:47:12 +0800 Subject: [PATCH 7/9] feat: assistant connect --- backend/apps/datasource/crud/datasource.py | 2 +- backend/apps/system/crud/assistant.py | 86 ++++++++++++++-------- 2 files changed, 57 insertions(+), 31 deletions(-) diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index a73a22bf..ed52dfa4 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -41,7 +41,7 @@ def get_ds(session: SessionDep, id: int): def check_status(session: SessionDep, ds: CoreDatasource, is_raise: bool = False): - conn = get_engine(ds, 5) + conn = get_engine(ds, 10) try: with conn.connect() as connection: SQLBotLogUtil.info("success") diff --git a/backend/apps/system/crud/assistant.py b/backend/apps/system/crud/assistant.py index 9b1579fc..72bdfa57 100644 --- a/backend/apps/system/crud/assistant.py +++ b/backend/apps/system/crud/assistant.py @@ -1,29 +1,33 @@ - - import json +import urllib from typing import Optional -from fastapi import FastAPI + import requests +from fastapi import FastAPI from sqlalchemy import Engine, create_engine from sqlmodel import Session, select -from apps.datasource.models.datasource import CoreDatasource, DatasourceConf +from starlette.middleware.cors import CORSMiddleware +from apps.datasource.models.datasource import CoreDatasource, DatasourceConf from apps.system.models.system_model import AssistantModel from apps.system.schemas.auth import CacheName, CacheNamespace from apps.system.schemas.system_schema import AssistantHeader, AssistantOutDsSchema, UserInfoDTO -from common.core.sqlbot_cache import cache -from common.core.db import engine -from starlette.middleware.cors import CORSMiddleware from common.core.config import settings +from common.core.db import engine +from common.core.sqlbot_cache import cache from common.utils.utils import string_to_numeric_hash + @cache(namespace=CacheNamespace.EMBEDDED_INFO, cacheName=CacheName.ASSISTANT_INFO, keyExpression="assistant_id") async def get_assistant_info(*, session: Session, assistant_id: int) -> AssistantModel | None: db_model = session.get(AssistantModel, assistant_id) return db_model + def get_assistant_user(*, id: int): - return UserInfoDTO(id=id, account="sqlbot-inner-assistant", oid=1, name="sqlbot-inner-assistant", email="sqlbot-inner-assistant@sqlbot.com") + return UserInfoDTO(id=id, account="sqlbot-inner-assistant", oid=1, name="sqlbot-inner-assistant", + email="sqlbot-inner-assistant@sqlbot.com") + def get_assistant_ds(llm_service) -> list[dict]: assistant: AssistantHeader = llm_service.current_assistant @@ -34,13 +38,14 @@ def get_assistant_ds(llm_service) -> list[dict]: if configuration: config: dict[any] = json.loads(configuration) oid: int = int(config['oid']) - stmt = select(CoreDatasource.id, CoreDatasource.name, CoreDatasource.description).where(CoreDatasource.oid == oid) + stmt = select(CoreDatasource.id, CoreDatasource.name, CoreDatasource.description).where( + CoreDatasource.oid == oid) if not assistant.online: - private_list:list[int] = config.get('private_list') or None + private_list: list[int] = config.get('private_list') or None if private_list: stmt = stmt.where(~CoreDatasource.id.in_(private_list)) db_ds_list = session.exec(stmt) - + result_list = [ { "id": ds.id, @@ -49,7 +54,7 @@ def get_assistant_ds(llm_service) -> list[dict]: } for ds in db_ds_list ] - + # filter private ds if offline return result_list out_ds_instance: AssistantOutDs = AssistantOutDsFactory.get_instance(assistant) @@ -58,8 +63,9 @@ def get_assistant_ds(llm_service) -> list[dict]: # format? return dslist + def init_dynamic_cors(app: FastAPI): - try: + try: with Session(engine) as session: list_result = session.exec(select(AssistantModel).order_by(AssistantModel.create_time)).all() seen = set() @@ -81,20 +87,20 @@ def init_dynamic_cors(app: FastAPI): cors_middleware.kwargs['allow_origins'] = updated_origins except Exception as e: return False, e - - + class AssistantOutDs: assistant: AssistantHeader ds_list: Optional[list[AssistantOutDsSchema]] = None certificate: Optional[str] = None + def __init__(self, assistant: AssistantHeader): self.assistant = assistant self.ds_list = None self.certificate = assistant.certificate self.get_ds_from_api() - - #@cache(namespace=CacheNamespace.EMBEDDED_INFO, cacheName=CacheName.ASSISTANT_DS, keyExpression="current_user.id") + + # @cache(namespace=CacheNamespace.EMBEDDED_INFO, cacheName=CacheName.ASSISTANT_DS, keyExpression="current_user.id") def get_ds_from_api(self): config: dict[any] = json.loads(self.assistant.configuration) endpoint: str = config['endpoint'] @@ -118,23 +124,23 @@ class AssistantOutDs: self.convert2schema(item) for item in temp_list ] - + return self.ds_list else: raise Exception(f"Failed to get datasource list from {endpoint}, error: {result_json.get('message')}") else: raise Exception(f"Failed to get datasource list from {endpoint}, status code: {res.status_code}") - + def get_simple_ds_list(self): if self.ds_list: return [{'id': ds.id, 'name': ds.name, 'description': ds.comment} for ds in self.ds_list] else: raise Exception("Datasource list is not found.") - + def get_db_schema(self, ds_id: int) -> str: ds = self.get_ds(ds_id) schema_str = "" - #db_name = ds.db_schema + # db_name = ds.db_schema db_name = ds.db_schema if ds.db_schema is not None and ds.db_schema != "" else ds.dataBase schema_str += f"【DB_ID】 {db_name}\n【Schema】\n" for table in ds.tables: @@ -144,7 +150,7 @@ class AssistantOutDs: schema_str += '\n[\n' else: schema_str += f", {table_comment}\n[\n" - + field_list = [] for field in table.fields: field_comment = field.comment @@ -155,7 +161,7 @@ class AssistantOutDs: schema_str += ",\n".join(field_list) schema_str += '\n]\n' return schema_str - + def get_ds(self, ds_id: int): if self.ds_list: for ds in self.ds_list: @@ -175,20 +181,22 @@ class AssistantOutDs: db_schema = ds_dict.get('schema', ds_dict.get('db_schema', '')) ds_dict.pop("schema", None) return AssistantOutDsSchema(**{**ds_dict, "id": id, "db_schema": db_schema}) - + + class AssistantOutDsFactory: @staticmethod def get_instance(assistant: AssistantHeader) -> AssistantOutDs: return AssistantOutDs(assistant) + def get_ds_engine(ds: AssistantOutDsSchema) -> Engine: timeout: int = 30 connect_args = {"connect_timeout": timeout} conf = DatasourceConf( - host=ds.host, - port=ds.port, + host=ds.host, + port=ds.port, username=ds.user, - password=ds.password, + password=ds.password, database=ds.dataBase, driver='', extraJdbc=ds.extraParams, @@ -197,8 +205,26 @@ def get_ds_engine(ds: AssistantOutDsSchema) -> Engine: conf.extraJdbc = '' from apps.db.db import get_uri_from_config uri = get_uri_from_config(ds.type, conf) + # if ds.type == "pg" and ds.db_schema: + # connect_args.update({"options": f"-c search_path={ds.db_schema}"}) + # engine = create_engine(uri, connect_args=connect_args, pool_timeout=timeout, pool_size=20, max_overflow=10) + if ds.type == "pg" and ds.db_schema: - connect_args.update({"options": f"-c search_path={ds.db_schema}"}) - engine = create_engine(uri, connect_args=connect_args, pool_timeout=timeout, pool_size=20, max_overflow=10) + engine = create_engine(uri, + connect_args={"options": f"-c search_path={urllib.parse.quote(ds.db_schema)}", + "connect_timeout": timeout}, + pool_timeout=timeout, pool_size=20, max_overflow=10) + elif ds.type == 'sqlServer': + engine = create_engine(uri, pool_timeout=timeout, + pool_size=20, + max_overflow=10) + elif ds.type == 'oracle': + engine = create_engine(uri, + pool_timeout=timeout, + pool_size=20, + max_overflow=10) + else: + engine = create_engine(uri, connect_args={"connect_timeout": timeout}, pool_timeout=timeout, + pool_size=20, + max_overflow=10) return engine - \ No newline at end of file From f6253cdb0ce6c2d129d0db532895965507ab84f1 Mon Sep 17 00:00:00 2001 From: junjun Date: Tue, 29 Jul 2025 12:00:03 +0800 Subject: [PATCH 8/9] feat: connect --- backend/apps/db/db.py | 14 ++++---------- backend/apps/db/engine.py | 4 +--- backend/apps/system/crud/assistant.py | 14 ++++---------- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index 83be9c75..ede3d277 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -60,20 +60,14 @@ def get_engine(ds: CoreDatasource, timeout: int = 30) -> Engine: engine = create_engine(get_uri(ds), connect_args={"options": f"-c search_path={urllib.parse.quote(conf.dbSchema)}", "connect_timeout": conf.timeout}, - pool_timeout=conf.timeout, pool_size=20, max_overflow=10) + pool_timeout=conf.timeout) elif ds.type == 'sqlServer': - engine = create_engine(get_uri(ds), pool_timeout=conf.timeout, - pool_size=20, - max_overflow=10) + engine = create_engine(get_uri(ds), pool_timeout=conf.timeout) elif ds.type == 'oracle': engine = create_engine(get_uri(ds), - pool_timeout=conf.timeout, - pool_size=20, - max_overflow=10) + pool_timeout=conf.timeout) else: - engine = create_engine(get_uri(ds), connect_args={"connect_timeout": conf.timeout}, pool_timeout=conf.timeout, - pool_size=20, - max_overflow=10) + engine = create_engine(get_uri(ds), connect_args={"connect_timeout": conf.timeout}, pool_timeout=conf.timeout) return engine diff --git a/backend/apps/db/engine.py b/backend/apps/db/engine.py index e86756f5..f4c95274 100644 --- a/backend/apps/db/engine.py +++ b/backend/apps/db/engine.py @@ -25,9 +25,7 @@ def get_engine_conn(): db_url = get_engine_uri(conf) engine = create_engine(db_url, connect_args={"options": f"-c search_path={conf.dbSchema}", "connect_timeout": conf.timeout}, - pool_timeout=conf.timeout, - pool_size=20, - max_overflow=10) + pool_timeout=conf.timeout) return engine diff --git a/backend/apps/system/crud/assistant.py b/backend/apps/system/crud/assistant.py index 72bdfa57..090c4f1b 100644 --- a/backend/apps/system/crud/assistant.py +++ b/backend/apps/system/crud/assistant.py @@ -213,18 +213,12 @@ def get_ds_engine(ds: AssistantOutDsSchema) -> Engine: engine = create_engine(uri, connect_args={"options": f"-c search_path={urllib.parse.quote(ds.db_schema)}", "connect_timeout": timeout}, - pool_timeout=timeout, pool_size=20, max_overflow=10) + pool_timeout=timeout) elif ds.type == 'sqlServer': - engine = create_engine(uri, pool_timeout=timeout, - pool_size=20, - max_overflow=10) + engine = create_engine(uri, pool_timeout=timeout) elif ds.type == 'oracle': engine = create_engine(uri, - pool_timeout=timeout, - pool_size=20, - max_overflow=10) + pool_timeout=timeout) else: - engine = create_engine(uri, connect_args={"connect_timeout": timeout}, pool_timeout=timeout, - pool_size=20, - max_overflow=10) + engine = create_engine(uri, connect_args={"connect_timeout": timeout}, pool_timeout=timeout) return engine From 91cd477e3563f69ee8f6071f69887be7a5804f40 Mon Sep 17 00:00:00 2001 From: junjun Date: Tue, 29 Jul 2025 12:14:39 +0800 Subject: [PATCH 9/9] feat: connect --- backend/apps/db/db.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index ede3d277..607d7cb0 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -52,10 +52,12 @@ def get_uri_from_config(type: str, conf: DatasourceConf) -> str: return db_url -def get_engine(ds: CoreDatasource, timeout: int = 30) -> Engine: +def get_engine(ds: CoreDatasource, timeout: int = 0) -> Engine: conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) if ds.type != "excel" else get_engine_config() if conf.timeout is None: conf.timeout = timeout + if timeout > 0: + conf.timeout = timeout if ds.type == "pg" and (conf.dbSchema is not None and conf.dbSchema != ""): engine = create_engine(get_uri(ds), connect_args={"options": f"-c search_path={urllib.parse.quote(conf.dbSchema)}",