From e50b71af2cce98ce441a6e452098cdeabdb911da Mon Sep 17 00:00:00 2001 From: junjun Date: Mon, 19 May 2025 13:38:01 +0800 Subject: [PATCH] feat(datasource): Support Mssql --- backend/apps/datasource/crud/datasource.py | 10 +++ backend/apps/datasource/models/datasource.py | 1 + backend/apps/db/db.py | 81 +++++++++++++++----- backend/pyproject.toml | 3 +- frontend/src/views/ds/form.vue | 22 ++++-- frontend/src/views/ds/index.vue | 2 + 6 files changed, 94 insertions(+), 25 deletions(-) diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index 94ff11c7..70b94b3f 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -132,6 +132,9 @@ def sync_table(session: SessionDep, ds: CoreDatasource, tables: List[CoreTable]) if len(id_list) > 0: session.query(CoreTable).filter(and_(CoreTable.ds_id == ds.id, CoreTable.id.not_in(id_list))).delete( synchronize_session=False) + session.query(CoreField).filter(and_(CoreField.ds_id == ds.id, CoreField.table_id.not_in(id_list))).delete( + synchronize_session=False) + session.commit() def sync_fields(session: SessionDep, ds: CoreDatasource, table: CoreTable, fields: List[ColumnSchema]): @@ -161,6 +164,7 @@ def sync_fields(session: SessionDep, ds: CoreDatasource, table: CoreTable, field if len(id_list) > 0: session.query(CoreField).filter(and_(CoreField.table_id == table.id, CoreField.id.not_in(id_list))).delete( synchronize_session=False) + session.commit() def update_table_and_fields(session: SessionDep, data: EditObj): @@ -174,4 +178,10 @@ def preview(session: SessionDep, id: int, data: EditObj): sql: str = "" if ds.type == "mysql": sql = f"""SELECT {", ".join([f.field_name for f in data.fields if f.checked])} FROM {data.table.table_name} LIMIT 100""" + elif ds.type == "sqlServer": + sql = f""" + SELECT {", ".join([f.field_name for f in data.fields if f.checked])} FROM {data.table.table_name} + ORDER BY {data.fields[0].field_name} + OFFSET 0 ROWS FETCH NEXT 100 ROWS ONLY + """ return exec_sql(ds, sql) diff --git a/backend/apps/datasource/models/datasource.py b/backend/apps/datasource/models/datasource.py index 18352d97..ce8caaa0 100644 --- a/backend/apps/datasource/models/datasource.py +++ b/backend/apps/datasource/models/datasource.py @@ -67,6 +67,7 @@ class DatasourceConf(BaseModel): database: str = '' driver: str = '' extraJdbc: str = '' + dbSchema: str = '' class TableSchema: diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index 24d4ca38..4f5a96d1 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -17,6 +17,11 @@ def get_uri(ds: CoreDatasource): db_url = f"mysql+pymysql://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}?{urllib.parse.quote(conf.extraJdbc)}" else: db_url = f"mysql+pymysql://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}" + elif ds.type == "sqlServer": + if conf.extraJdbc is not None and conf.extraJdbc != '': + db_url = f"mssql+pymssql://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}?{urllib.parse.quote(conf.extraJdbc)}" + else: + db_url = f"mssql+pymssql://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}" else: raise 'The datasource type not support.' return db_url @@ -33,17 +38,36 @@ def get_tables(ds: CoreDatasource): conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) session = get_session(ds) result: Result[Any] + sql: str = '' try: if ds.type == "mysql": - sql = f"""SELECT - TABLE_NAME, - TABLE_COMMENT - FROM - information_schema.TABLES - WHERE - TABLE_SCHEMA = '{conf.database}';""" - result = session.execute(text(sql)) + sql = f""" + SELECT + TABLE_NAME, + TABLE_COMMENT + FROM + information_schema.TABLES + WHERE + TABLE_SCHEMA = '{conf.database}'; + """ + elif ds.type == "sqlServer": + sql = f""" + SELECT + TABLE_NAME AS [TABLE_NAME], + ISNULL(ep.value, '') AS [TABLE_COMMENT] + FROM + INFORMATION_SCHEMA.TABLES t + LEFT JOIN + sys.extended_properties ep + ON ep.major_id = OBJECT_ID(t.TABLE_SCHEMA + '.' + t.TABLE_NAME) + AND ep.minor_id = 0 + AND ep.name = 'MS_Description' + WHERE + t.TABLE_TYPE IN ('BASE TABLE', 'VIEW') + AND t.TABLE_SCHEMA = '{conf.dbSchema}'; + """ + result = session.execute(text(sql)) res = result.fetchall() res_list = [TableSchema(*item) for item in res] return res_list @@ -58,20 +82,41 @@ def get_fields(ds: CoreDatasource, table_name: str = None): conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) session = get_session(ds) result: Result[Any] + sql: str = '' try: if ds.type == "mysql": - sql1 = f"""SELECT - COLUMN_NAME, - DATA_TYPE, - COLUMN_COMMENT - FROM - INFORMATION_SCHEMA.COLUMNS - WHERE - TABLE_SCHEMA = '{conf.database}'""" - sql2 = f""" AND TABLE_NAME = '{table_name}';""" if table_name is not None and table_name != "" else ";" + sql1 = f""" + SELECT + COLUMN_NAME, + DATA_TYPE, + COLUMN_COMMENT + FROM + INFORMATION_SCHEMA.COLUMNS + WHERE + TABLE_SCHEMA = '{conf.database}' + """ + sql2 = f" AND TABLE_NAME = '{table_name}';" if table_name is not None and table_name != "" else ";" + sql = sql1 + sql2 + elif ds.type == "sqlServer": + sql1 = f""" + SELECT + COLUMN_NAME AS [COLUMN_NAME], + DATA_TYPE AS [DATA_TYPE], + ISNULL(EP.value, '') AS [COLUMN_COMMENT] + FROM + INFORMATION_SCHEMA.COLUMNS C + LEFT JOIN + sys.extended_properties EP + ON EP.major_id = OBJECT_ID(C.TABLE_SCHEMA + '.' + C.TABLE_NAME) + AND EP.minor_id = C.ORDINAL_POSITION + AND EP.name = 'MS_Description' + WHERE + C.TABLE_SCHEMA = '{conf.dbSchema}' + """ + sql2 = f"AND C.TABLE_NAME = '{table_name}';" if table_name is not None and table_name != "" else ";" sql = sql1 + sql2 - result = session.execute(text(sql)) + result = session.execute(text(sql)) res = result.fetchall() res_list = [ColumnSchema(*item) for item in res] return res_list diff --git a/backend/pyproject.toml b/backend/pyproject.toml index e1443477..a5e85165 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -28,7 +28,8 @@ dependencies = [ "dashscope>=1.14.0,<2.0.0", "pymysql (>=1.1.1,<2.0.0)", "cryptography (>=44.0.3,<45.0.0)", - "llama_index>=0.12.35" + "llama_index>=0.12.35", + "pymssql (>=2.3.4,<3.0.0)" ] [[tool.uv.index]] url = "https://pypi.tuna.tsinghua.edu.cn/simple" diff --git a/frontend/src/views/ds/form.vue b/frontend/src/views/ds/form.vue index d21094b2..dd50d0e9 100644 --- a/frontend/src/views/ds/form.vue +++ b/frontend/src/views/ds/form.vue @@ -20,7 +20,7 @@ - + + + + Get Schema +
- + @@ -76,6 +80,7 @@ import { datasourceApi } from '@/api/datasource' import { encrypted, decrypted } from './js/aes' import { ElMessage } from 'element-plus' import type { FormInstance, FormRules } from 'element-plus' +import { Plus } from '@element-plus/icons-vue' const dsFormRef = ref() const emit = defineEmits(['refresh']) @@ -94,7 +99,8 @@ const rules = reactive({ const dialogVisible = ref(false) const dsType = [ - {label:"MySQL", value:"mysql"} + {label:"MySQL", value:"mysql"}, + {label:"SQL Server", value:"sqlServer"} ] const form = ref({ name:'', @@ -109,7 +115,8 @@ const config = ref({ username:'', password:'', database:'', - extraJdbc:'' + extraJdbc:'', + dbSchema:'' }) const close = () => { @@ -138,6 +145,7 @@ const open = (item: any, editTable: boolean = false) => { config.value.password = configuration.password config.value.database = configuration.database config.value.extraJdbc = configuration.extraJdbc + config.value.dbSchema = configuration.dbSchema } if (editTable) { @@ -171,6 +179,7 @@ const open = (item: any, editTable: boolean = false) => { password:'', database:'', extraJdbc:'', + dbSchema:'' } } dialogVisible.value = true @@ -218,7 +227,8 @@ const buildConf = () => { username:config.value.username, password:config.value.password, database:config.value.database, - extraJdbc:config.value.extraJdbc + extraJdbc:config.value.extraJdbc, + dbSchema:config.value.dbSchema })) } @@ -262,7 +272,7 @@ defineExpose({ open }) \ No newline at end of file diff --git a/frontend/src/views/ds/index.vue b/frontend/src/views/ds/index.vue index 2e0cbdd5..e99f6e99 100644 --- a/frontend/src/views/ds/index.vue +++ b/frontend/src/views/ds/index.vue @@ -27,6 +27,7 @@
+
@@ -52,6 +53,7 @@ import IconOpeAdd from '@/assets/svg/operate/ope-add.svg' import IconOpeEdit from '@/assets/svg/operate/ope-edit.svg' import IconOpeDelete from '@/assets/svg/operate/ope-delete.svg' import MysqlDs from '@/assets/svg/ds/mysql-ds.svg' +import SQLServerDs from '@/assets/svg/ds/sqlServer-ds.svg' import { Search, List, CreditCard } from '@element-plus/icons-vue' import { ref, onMounted } from 'vue' import DsForm from './form.vue'