feat(datasource): commit any code for excel

backend/apps/datasource/api/datasource.py

@@ -1,4 +1,4 @@
-from fastapi import APIRouter, File, UploadFile
+from fastapi import APIRouter, File, UploadFile, HTTPException
 from ..crud.datasource import get_datasource_list, check_status, create_ds, update_ds, delete_ds, getTables, getFields, \
     execSql, update_table_and_fields, getTablesByDs, chooseTables, preview
 from common.core.deps import SessionDep
@@ -6,6 +6,9 @@ from ..models.datasource import CoreDatasource, CreateDatasource, EditObj, CoreT
 from ..crud.table import get_tables_by_ds_id
 from ..crud.field import get_fields_by_table_id
 from typing import List
+import uuid
+import pandas as pd
+import hashlib
 
 router = APIRouter(tags=["datasource"], prefix="/datasource")
 
@@ -80,7 +83,27 @@ async def edit_local(session: SessionDep, id: int, data: EditObj):
     return preview(session, id, data)
 
 
-@router.post("/uploadExcel/")
+@router.post("/uploadExcel")
 async def upload_excel(session: SessionDep, file: UploadFile = File(...)):
+    ALLOWED_EXTENSIONS = {"xlsx", "xls", "csv"}
+    if not file.filename.lower().endswith(tuple(ALLOWED_EXTENSIONS)):
+        raise HTTPException(400, "Only support .xlsx/.xls/.csv")
+
+    contents = await file.read()
+    df_sheets = pd.read_excel(contents, sheet_name=None)
+    # build columns and data to insert db
     # todo
-    pass
+    sheets = []
+    for sheet_name, df in df_sheets.items():
+        print("--------------------")
+        print(f"Sheet: {sheet_name}, {hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}")
+        sheets.append({"name": f"{sheet_name}",
+                       "uniqueName": f"{sheet_name}_{hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}"})
+        column_len = len(df.dtypes)
+        for i in range(column_len):
+            print(f"{df.columns[i]} , {df.dtypes[i]}")
+
+        for row in df.values:
+            print(row)
+
+    return {"filename": file.filename, "sheets": sheets}

backend/apps/db/engine.py

@@ -0,0 +1,2 @@
+# Author: Junjun
+# Date: 2025/5/19

backend/pyproject.toml

@@ -29,7 +29,9 @@ dependencies = [
     "pymysql (>=1.1.1,<2.0.0)",
     "cryptography (>=44.0.3,<45.0.0)",
     "llama_index>=0.12.35",
-    "pymssql (>=2.3.4,<3.0.0)"
+    "pymssql (>=2.3.4,<3.0.0)",
+    "pandas (>=2.2.3,<3.0.0)",
+    "openpyxl (>=3.1.5,<4.0.0)"
 ]
 [[tool.uv.index]]
 url = "https://pypi.tuna.tsinghua.edu.cn/simple"

frontend/src/views/ds/form.vue

@@ -29,28 +29,47 @@
             />
           </el-select>
         </el-form-item>
-        <el-form-item label="Host/Ip">
-          <el-input v-model="config.host" />
-        </el-form-item>
-        <el-form-item label="Port">
-          <el-input v-model="config.port" />
-        </el-form-item>
-        <el-form-item label="Username">
-          <el-input v-model="config.username" />
-        </el-form-item>
-        <el-form-item label="Password">
-          <el-input v-model="config.password" type="password" show-password />
-        </el-form-item>
-        <el-form-item label="Database">
-          <el-input v-model="config.database" />
-        </el-form-item>
-        <el-form-item label="Extra JDBC String">
-          <el-input v-model="config.extraJdbc" />
-        </el-form-item>
-        <el-form-item label="Schema" v-if="form.type === 'sqlServer'">
-          <el-input v-model="config.dbSchema" />
-          <el-button link type="primary" :icon="Plus" v-if="false">Get Schema</el-button>
-        </el-form-item>
+        <div v-if="form.type === 'excel'">
+          <el-form-item label="File">
+            <el-upload
+              accept=".xls, .xlsx, .csv"
+              :headers="headers"
+              action="http://localhost:8000/api/v1/datasource/uploadExcel"
+              :before-upload="beforeUpload"
+            >
+              <el-button>Upload</el-button>
+              <template #tip>
+                <div class="el-upload__tip">
+                  Only support .xls, .xlsx, .csv, size less than 50MB.
+                </div>
+              </template>
+            </el-upload>
+          </el-form-item>
+        </div>
+        <div v-else>
+          <el-form-item label="Host/Ip">
+            <el-input v-model="config.host" />
+          </el-form-item>
+          <el-form-item label="Port">
+            <el-input v-model="config.port" />
+          </el-form-item>
+          <el-form-item label="Username">
+            <el-input v-model="config.username" />
+          </el-form-item>
+          <el-form-item label="Password">
+            <el-input v-model="config.password" type="password" show-password />
+          </el-form-item>
+          <el-form-item label="Database">
+            <el-input v-model="config.database" />
+          </el-form-item>
+          <el-form-item label="Extra JDBC String">
+            <el-input v-model="config.extraJdbc" />
+          </el-form-item>
+          <el-form-item label="Schema" v-if="form.type === 'sqlServer'">
+            <el-input v-model="config.dbSchema" />
+            <el-button link type="primary" :icon="Plus" v-if="false">Get Schema</el-button>
+          </el-form-item>
+        </div>
       </el-form>
     </div>
     <div v-show="active === 2" class="container">
@@ -67,7 +86,7 @@
     </div>
     <div style="display: flex;justify-content: flex-end;margin-top: 20px;">
       <el-button @click="close">Cancel</el-button>
-      <el-button v-show="!isCreate && !isEditTable" @click="check">Test Connect</el-button>
+      <el-button v-show="!isCreate && !isEditTable && form.type !== 'excel'" @click="check">Test Connect</el-button>
       <el-button v-show="active === 1 && isCreate" type="primary" @click="next(dsFormRef)">Next</el-button>
       <el-button v-show="active === 2 && isCreate" @click="preview">Preview</el-button>
       <el-button v-show="active === 2 || !isCreate" type="primary" @click="save(dsFormRef)">Save</el-button>
@@ -81,7 +100,9 @@ import { encrypted, decrypted } from './js/aes'
 import { ElMessage } from 'element-plus'
 import type { FormInstance, FormRules } from 'element-plus'
 import { Plus } from '@element-plus/icons-vue'
+import { useCache } from '@/utils/useCache'
 
+const { wsCache } = useCache()
 const dsFormRef = ref<FormInstance>()
 const emit = defineEmits(['refresh'])
 const active = ref(1)
@@ -90,6 +111,10 @@ const isEditTable = ref(false)
 const checkList = ref<any>([])
 const tableList = ref<any>([])
 
+const token = wsCache.get('user.token')
+const headers = ref<any>({'X-SQLBOT-TOKEN': `Bearer ${token}`})
+
+
 const rules = reactive<FormRules>({
   name: [
     { required: true, message: 'Please input name', trigger: 'blur' },
@@ -99,6 +124,7 @@ const rules = reactive<FormRules>({
 
 const dialogVisible = ref<boolean>(false)
 const dsType = [
+  {label:"Excel/CSV", value:"excel"},
   {label:"MySQL", value:"mysql"},
   {label:"SQL Server", value:"sqlServer"}
 ]
@@ -246,28 +272,40 @@ const check = () => {
 }
 
 const next = async(formEl: FormInstance | undefined) => {
-  if(!formEl) return
-  await formEl.validate((valid) => {
-    if (valid) {
-      // check status if success do next
-      buildConf()
-      datasourceApi.check(form.value).then((res: boolean) => {
-        if(res) {
-          active.value++
-          // request tables
-          datasourceApi.getTablesByConf(form.value).then((res) => {
-            tableList.value = res
-          })
-        }
-      })
-    }
-  })
+  if (form.value.type === "excel") {
+    // next, show tables 
+  } else {
+    if(!formEl) return
+    await formEl.validate((valid) => {
+      if (valid) {
+        // check status if success do next
+        buildConf()
+        datasourceApi.check(form.value).then((res: boolean) => {
+          if(res) {
+            active.value++
+            // request tables
+            datasourceApi.getTablesByConf(form.value).then((res) => {
+              tableList.value = res
+            })
+          }
+        })
+      }
+    })
+  }
 }
 
 const preview = () => {
   active.value--
 }
 
+const beforeUpload = (rawFile: any) => {
+  if (rawFile.size / 1024 / 1024 > 50) {
+    ElMessage.error('File size can not exceed 50MB!')
+    return false
+  }
+  return true
+}
+
 defineExpose({ open })
 </script>
 <style lang="less" scoped>