From e61ead238802011cd7df516a8d5493ad7fd86142 Mon Sep 17 00:00:00 2001 From: junjun Date: Mon, 19 May 2025 17:46:48 +0800 Subject: [PATCH] feat(datasource): commit any code for excel --- backend/apps/datasource/api/datasource.py | 29 +++++- backend/apps/db/engine.py | 2 + backend/pyproject.toml | 4 +- frontend/src/views/ds/form.vue | 116 ++++++++++++++-------- 4 files changed, 108 insertions(+), 43 deletions(-) create mode 100644 backend/apps/db/engine.py diff --git a/backend/apps/datasource/api/datasource.py b/backend/apps/datasource/api/datasource.py index afd7595d..37e555f8 100644 --- a/backend/apps/datasource/api/datasource.py +++ b/backend/apps/datasource/api/datasource.py @@ -1,4 +1,4 @@ -from fastapi import APIRouter, File, UploadFile +from fastapi import APIRouter, File, UploadFile, HTTPException from ..crud.datasource import get_datasource_list, check_status, create_ds, update_ds, delete_ds, getTables, getFields, \ execSql, update_table_and_fields, getTablesByDs, chooseTables, preview from common.core.deps import SessionDep @@ -6,6 +6,9 @@ from ..models.datasource import CoreDatasource, CreateDatasource, EditObj, CoreT from ..crud.table import get_tables_by_ds_id from ..crud.field import get_fields_by_table_id from typing import List +import uuid +import pandas as pd +import hashlib router = APIRouter(tags=["datasource"], prefix="/datasource") @@ -80,7 +83,27 @@ async def edit_local(session: SessionDep, id: int, data: EditObj): return preview(session, id, data) -@router.post("/uploadExcel/") +@router.post("/uploadExcel") async def upload_excel(session: SessionDep, file: UploadFile = File(...)): + ALLOWED_EXTENSIONS = {"xlsx", "xls", "csv"} + if not file.filename.lower().endswith(tuple(ALLOWED_EXTENSIONS)): + raise HTTPException(400, "Only support .xlsx/.xls/.csv") + + contents = await file.read() + df_sheets = pd.read_excel(contents, sheet_name=None) + # build columns and data to insert db # todo - pass + sheets = [] + for sheet_name, df in df_sheets.items(): + print("--------------------") + print(f"Sheet: {sheet_name}, {hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}") + sheets.append({"name": f"{sheet_name}", + "uniqueName": f"{sheet_name}_{hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}"}) + column_len = len(df.dtypes) + for i in range(column_len): + print(f"{df.columns[i]} , {df.dtypes[i]}") + + for row in df.values: + print(row) + + return {"filename": file.filename, "sheets": sheets} diff --git a/backend/apps/db/engine.py b/backend/apps/db/engine.py new file mode 100644 index 00000000..1d125831 --- /dev/null +++ b/backend/apps/db/engine.py @@ -0,0 +1,2 @@ +# Author: Junjun +# Date: 2025/5/19 diff --git a/backend/pyproject.toml b/backend/pyproject.toml index a5e85165..d165de11 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -29,7 +29,9 @@ dependencies = [ "pymysql (>=1.1.1,<2.0.0)", "cryptography (>=44.0.3,<45.0.0)", "llama_index>=0.12.35", - "pymssql (>=2.3.4,<3.0.0)" + "pymssql (>=2.3.4,<3.0.0)", + "pandas (>=2.2.3,<3.0.0)", + "openpyxl (>=3.1.5,<4.0.0)" ] [[tool.uv.index]] url = "https://pypi.tuna.tsinghua.edu.cn/simple" diff --git a/frontend/src/views/ds/form.vue b/frontend/src/views/ds/form.vue index dd50d0e9..009b485c 100644 --- a/frontend/src/views/ds/form.vue +++ b/frontend/src/views/ds/form.vue @@ -29,28 +29,47 @@ /> - - - - - - - - - - - - - - - - - - - - - Get Schema - +
+ + + Upload + + + +
+
+ + + + + + + + + + + + + + + + + + + + + Get Schema + +
@@ -67,7 +86,7 @@
Cancel - Test Connect + Test Connect Next Preview Save @@ -81,7 +100,9 @@ import { encrypted, decrypted } from './js/aes' import { ElMessage } from 'element-plus' import type { FormInstance, FormRules } from 'element-plus' import { Plus } from '@element-plus/icons-vue' +import { useCache } from '@/utils/useCache' +const { wsCache } = useCache() const dsFormRef = ref() const emit = defineEmits(['refresh']) const active = ref(1) @@ -90,6 +111,10 @@ const isEditTable = ref(false) const checkList = ref([]) const tableList = ref([]) +const token = wsCache.get('user.token') +const headers = ref({'X-SQLBOT-TOKEN': `Bearer ${token}`}) + + const rules = reactive({ name: [ { required: true, message: 'Please input name', trigger: 'blur' }, @@ -99,6 +124,7 @@ const rules = reactive({ const dialogVisible = ref(false) const dsType = [ + {label:"Excel/CSV", value:"excel"}, {label:"MySQL", value:"mysql"}, {label:"SQL Server", value:"sqlServer"} ] @@ -246,28 +272,40 @@ const check = () => { } const next = async(formEl: FormInstance | undefined) => { - if(!formEl) return - await formEl.validate((valid) => { - if (valid) { - // check status if success do next - buildConf() - datasourceApi.check(form.value).then((res: boolean) => { - if(res) { - active.value++ - // request tables - datasourceApi.getTablesByConf(form.value).then((res) => { - tableList.value = res - }) - } - }) - } - }) + if (form.value.type === "excel") { + // next, show tables + } else { + if(!formEl) return + await formEl.validate((valid) => { + if (valid) { + // check status if success do next + buildConf() + datasourceApi.check(form.value).then((res: boolean) => { + if(res) { + active.value++ + // request tables + datasourceApi.getTablesByConf(form.value).then((res) => { + tableList.value = res + }) + } + }) + } + }) + } } const preview = () => { active.value-- } +const beforeUpload = (rawFile: any) => { + if (rawFile.size / 1024 / 1024 > 50) { + ElMessage.error('File size can not exceed 50MB!') + return false + } + return true +} + defineExpose({ open })