From cb1180e0f722c69601398937e628423b47e41425 Mon Sep 17 00:00:00 2001 From: junjun Date: Wed, 21 May 2025 11:25:04 +0800 Subject: [PATCH 1/3] feat(datasource): Support csv --- backend/apps/datasource/api/datasource.py | 43 ++++++++++++++++++---- backend/apps/datasource/crud/datasource.py | 1 - frontend/src/views/ds/form.vue | 28 +++++++------- 3 files changed, 49 insertions(+), 23 deletions(-) diff --git a/backend/apps/datasource/api/datasource.py b/backend/apps/datasource/api/datasource.py index fbc50924..a468552b 100644 --- a/backend/apps/datasource/api/datasource.py +++ b/backend/apps/datasource/api/datasource.py @@ -1,10 +1,10 @@ import hashlib +import os import uuid from typing import List import pandas as pd from fastapi import APIRouter, File, UploadFile, HTTPException -from numba.core.cgutils import if_zero from apps.db.engine import create_table, get_data_engine, insert_data from common.core.deps import SessionDep @@ -15,6 +15,7 @@ from ..crud.table import get_tables_by_ds_id from ..models.datasource import CoreDatasource, CreateDatasource, EditObj, CoreTable router = APIRouter(tags=["datasource"], prefix="/datasource") +path = "/opt/sqlbot/data/excel" @router.get("/list") @@ -93,13 +94,17 @@ async def upload_excel(session: SessionDep, file: UploadFile = File(...)): if not file.filename.lower().endswith(tuple(ALLOWED_EXTENSIONS)): raise HTTPException(400, "Only support .xlsx/.xls/.csv") - contents = await file.read() - df_sheets = pd.read_excel(contents, sheet_name=None) - # build columns and data to insert db - sheets = [] + os.makedirs(path, exist_ok=True) + filename = f"{file.filename.split('.')[0]}_{hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}.{file.filename.split('.')[1]}" + save_path = os.path.join(path, filename) + with open(save_path, "wb") as f: + f.write(await file.read()) + conn = get_data_engine() - for sheet_name, df in df_sheets.items(): - tableName = f"{sheet_name}_{hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}" + sheets = [] + if filename.endswith(".csv"): + df = pd.read_csv(save_path) + tableName = f"sheet1_{hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}" sheets.append({"tableName": tableName, "tableComment": ""}) column_len = len(df.dtypes) fields = [] @@ -115,5 +120,27 @@ async def upload_excel(session: SessionDep, file: UploadFile = File(...)): ] # insert data insert_data(conn, tableName, fields, data) + else: + df_sheets = pd.read_excel(save_path, sheet_name=None) + # build columns and data to insert db + for sheet_name, df in df_sheets.items(): + tableName = f"{sheet_name}_{hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}" + sheets.append({"tableName": tableName, "tableComment": ""}) + column_len = len(df.dtypes) + fields = [] + for i in range(column_len): + # build fields + fields.append({"name": df.columns[i], "type": str(df.dtypes[i]), "relType": ""}) + # create table + create_table(conn, tableName, fields) + + data = [ + {df.columns[i]: int(row[i]) if "int" in str(df.dtypes[i]) else row[i] for i in range(len(row))} + for row in df.values + ] + # insert data + insert_data(conn, tableName, fields, data) conn.close() - return {"filename": file.filename, "sheets": sheets} + + os.remove(save_path) + return {"filename": filename, "sheets": sheets} diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index 3b5f0f9d..a3931e63 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -80,7 +80,6 @@ def delete_ds(session: SessionDep, id: int): for sheet in conf.sheets: conn.execute(text(f'DROP TABLE IF EXISTS "{sheet["tableName"]}"')) conn.commit() - conn.close() session.delete(term) session.commit() diff --git a/frontend/src/views/ds/form.vue b/frontend/src/views/ds/form.vue index 8b35c6a6..bfb4ecfa 100644 --- a/frontend/src/views/ds/form.vue +++ b/frontend/src/views/ds/form.vue @@ -189,8 +189,9 @@ const open = (item: any, editTable: boolean = false) => { if(item.type === 'excel') { tableList.value = config.value.sheets } else { - datasourceApi.getTablesByConf(form.value).then((res) => { - tableList.value = res + datasourceApi.getTablesByConf(form.value).then((table) => { + tableList.value = table + checkList.value = checkList.value.filter((ele:string) => {return table.map((ele:any) => {return ele.tableName}).includes(ele)}) }) } }) @@ -285,15 +286,15 @@ const check = () => { } const next = async(formEl: FormInstance | undefined) => { - if (form.value.type === "excel") { - // next, show tables - if (excelUploadSuccess.value) { - active.value++ - } - } else { - if(!formEl) return - await formEl.validate((valid) => { - if (valid) { + if(!formEl) return + await formEl.validate((valid) => { + if (valid) { + if (form.value.type === "excel") { + // next, show tables + if (excelUploadSuccess.value) { + active.value++ + } + } else { // check status if success do next buildConf() datasourceApi.check(form.value).then((res: boolean) => { @@ -306,8 +307,8 @@ const next = async(formEl: FormInstance | undefined) => { } }) } - }) - } + } + }) } const preview = () => { @@ -323,7 +324,6 @@ const beforeUpload = (rawFile: any) => { } const onSuccess = (response: any) => { - console.log(response) config.value.filename = response.data.filename config.value.sheets = response.data.sheets tableList.value = response.data.sheets From 1a295844afb8a6a4bb832c73c0a63dd139f3928a Mon Sep 17 00:00:00 2001 From: junjun Date: Wed, 21 May 2025 14:10:59 +0800 Subject: [PATCH 2/3] feat(datasource): datasource UI --- backend/apps/datasource/crud/datasource.py | 2 +- frontend/src/views/ds/form.vue | 12 ++++++++++++ frontend/src/views/ds/index.vue | 5 ++++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index a3931e63..0ae58661 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -17,7 +17,7 @@ from ..models.datasource import CoreDatasource, CreateDatasource, CoreTable, Cor DatasourceConf -def get_datasource_list(session: SessionDep) -> CoreDatasource: +def get_datasource_list(session: SessionDep): statement = select(CoreDatasource).order_by(CoreDatasource.create_time.desc()) datasource_list = session.exec(statement).fetchall() return datasource_list diff --git a/frontend/src/views/ds/form.vue b/frontend/src/views/ds/form.vue index bfb4ecfa..05f33039 100644 --- a/frontend/src/views/ds/form.vue +++ b/frontend/src/views/ds/form.vue @@ -281,6 +281,12 @@ const check = () => { type: 'success', showClose: true }) + } else { + ElMessage({ + message: 'Connect failed', + type: 'error', + showClose: true + }) } }) } @@ -304,6 +310,12 @@ const next = async(formEl: FormInstance | undefined) => { datasourceApi.getTablesByConf(form.value).then((res) => { tableList.value = res }) + } else { + ElMessage({ + message: 'Connect failed', + type: 'error', + showClose: true + }) } }) } diff --git a/frontend/src/views/ds/index.vue b/frontend/src/views/ds/index.vue index c47ce8a3..b73c7f8e 100644 --- a/frontend/src/views/ds/index.vue +++ b/frontend/src/views/ds/index.vue @@ -1,5 +1,5 @@