feat(datasource): Support Excel

backend/apps/datasource/api/datasource.py

@@ -1,14 +1,18 @@
+import hashlib
+import uuid
+from typing import List
+
+import pandas as pd
 from fastapi import APIRouter, File, UploadFile, HTTPException
+from numba.core.cgutils import if_zero
+
+from apps.db.engine import create_table, get_data_engine, insert_data
+from common.core.deps import SessionDep
 from ..crud.datasource import get_datasource_list, check_status, create_ds, update_ds, delete_ds, getTables, getFields, \
     execSql, update_table_and_fields, getTablesByDs, chooseTables, preview
-from common.core.deps import SessionDep
-from ..models.datasource import CoreDatasource, CreateDatasource, EditObj, CoreTable
-from ..crud.table import get_tables_by_ds_id
 from ..crud.field import get_fields_by_table_id
-from typing import List
-import uuid
-import pandas as pd
-import hashlib
+from ..crud.table import get_tables_by_ds_id
+from ..models.datasource import CoreDatasource, CreateDatasource, EditObj, CoreTable
 
 router = APIRouter(tags=["datasource"], prefix="/datasource")
 
@@ -92,18 +96,24 @@ async def upload_excel(session: SessionDep, file: UploadFile = File(...)):
     contents = await file.read()
     df_sheets = pd.read_excel(contents, sheet_name=None)
     # build columns and data to insert db
-    # todo
     sheets = []
+    conn = get_data_engine()
     for sheet_name, df in df_sheets.items():
-        print("--------------------")
-        print(f"Sheet: {sheet_name}, {hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}")
-        sheets.append({"name": f"{sheet_name}",
-                       "uniqueName": f"{sheet_name}_{hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}"})
+        tableName = f"{sheet_name}_{hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}"
+        sheets.append({"tableName": tableName, "tableComment": ""})
         column_len = len(df.dtypes)
+        fields = []
         for i in range(column_len):
-            print(f"{df.columns[i]} , {df.dtypes[i]}")
-
-        for row in df.values:
-            print(row)
+            # build fields
+            fields.append({"name": df.columns[i], "type": str(df.dtypes[i]), "relType": ""})
+        # create table
+        create_table(conn, tableName, fields)
 
+        data = [
+            {df.columns[i]: int(row[i]) if "int" in str(df.dtypes[i]) else row[i] for i in range(len(row))}
+            for row in df.values
+        ]
+        # insert data
+        insert_data(conn, tableName, fields, data)
+    conn.close()
     return {"filename": file.filename, "sheets": sheets}

backend/apps/datasource/crud/datasource.py

@@ -2,11 +2,13 @@ import datetime
 import json
 from typing import List
 
-from sqlalchemy import text, and_
+from sqlalchemy import and_, text
 from sqlmodel import select
 
 from apps.datasource.utils.utils import aes_decrypt
 from apps.db.db import get_engine, get_tables, get_fields, exec_sql
+from apps.db.engine import get_engine_config
+from apps.db.engine import get_engine_conn
 from common.core.deps import SessionDep
 from common.utils.utils import deepcopy_ignore_extra
 from ..crud.field import delete_field_by_ds_id, update_field
@@ -36,8 +38,8 @@ def create_ds(session: SessionDep, create_ds: CreateDatasource):
     ds = CoreDatasource()
     deepcopy_ignore_extra(create_ds, ds)
     ds.create_time = datetime.datetime.now()
-    status = check_status(session, ds)
-    ds.status = "Success" if status is True else "Fail"
+    # status = check_status(session, ds)
+    ds.status = "Success"
     record = CoreDatasource(**ds.model_dump())
     session.add(record)
     session.flush()
@@ -46,9 +48,7 @@ def create_ds(session: SessionDep, create_ds: CreateDatasource):
     session.commit()
 
     # save tables and fields
-    if status:
-        sync_table(session, ds, create_ds.tables)
-
+    sync_table(session, ds, create_ds.tables)
     return ds
 
 
@@ -72,6 +72,16 @@ def update_ds(session: SessionDep, ds: CoreDatasource):
 
 def delete_ds(session: SessionDep, id: int):
     term = session.exec(select(CoreDatasource).where(CoreDatasource.id == id)).first()
+    if term.type == "excel":
+        # drop all tables for current datasource
+        engine = get_engine_conn()
+        conf = DatasourceConf(**json.loads(aes_decrypt(term.configuration)))
+        with engine.connect() as conn:
+            for sheet in conf.sheets:
+                conn.execute(text(f'DROP TABLE IF EXISTS "{sheet["tableName"]}"'))
+            conn.commit()
+            conn.close()
+
     session.delete(term)
     session.commit()
     delete_table_by_ds_id(session, id)
@@ -98,6 +108,11 @@ def getFields(session: SessionDep, id: int, table_name: str):
     return fields
 
 
+def getFieldsByDs(session: SessionDep, ds: CoreDatasource, table_name: str):
+    fields = get_fields(ds, table_name)
+    return fields
+
+
 def execSql(session: SessionDep, id: int, sql: str):
     ds = session.exec(select(CoreDatasource).where(CoreDatasource.id == id)).first()
     return exec_sql(ds, sql)
@@ -128,7 +143,7 @@ def sync_table(session: SessionDep, ds: CoreDatasource, tables: List[CoreTable])
             session.commit()
 
         # sync field
-        fields = getFields(session, ds.id, item.table_name)
+        fields = getFieldsByDs(session, ds, item.table_name)
         sync_fields(session, ds, item, fields)
 
     if len(id_list) > 0:
@@ -177,7 +192,7 @@ def update_table_and_fields(session: SessionDep, data: EditObj):
 
 def preview(session: SessionDep, id: int, data: EditObj):
     ds = session.query(CoreDatasource).filter(CoreDatasource.id == id).first()
-    conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration)))
+    conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) if ds.type != "excel" else get_engine_config()
     sql: str = ""
     if ds.type == "mysql":
         sql = f"""SELECT `{"`, `".join([f.field_name for f in data.fields if f.checked])}` FROM `{data.table.table_name}` LIMIT 100"""
@@ -185,6 +200,6 @@ def preview(session: SessionDep, id: int, data: EditObj):
         sql = f"""SELECT [{"], [".join([f.field_name for f in data.fields if f.checked])}] FROM [{conf.dbSchema}].[{data.table.table_name}]
             ORDER BY [{data.fields[0].field_name}]
             OFFSET 0 ROWS FETCH NEXT 100 ROWS ONLY"""
-    elif ds.type == "pg":
+    elif ds.type == "pg" or ds.type == "excel":
         sql = f"""SELECT "{'", "'.join([f.field_name for f in data.fields if f.checked])}" FROM "{conf.dbSchema}"."{data.table.table_name}" LIMIT 100"""
     return exec_sql(ds, sql)

backend/apps/datasource/models/datasource.py

@@ -1,9 +1,10 @@
-from sqlmodel import SQLModel, Field
-from sqlalchemy import Column, Text, BigInteger, DateTime, Integer, Identity
 from datetime import datetime
-from pydantic import BaseModel
 from typing import List, Optional
 
+from pydantic import BaseModel
+from sqlalchemy import Column, Text, BigInteger, DateTime, Integer, Identity
+from sqlmodel import SQLModel, Field
+
 
 class CoreDatasource(SQLModel, table=True):
     __tablename__ = "core_datasource"
@@ -68,6 +69,22 @@ class DatasourceConf(BaseModel):
     driver: str = ''
     extraJdbc: str = ''
     dbSchema: str = ''
+    filename: str = ''
+    sheets: List = ''
+
+    def to_dict(self):
+        return {
+            "host": self.host,
+            "port": self.port,
+            "username": self.username,
+            "password": self.password,
+            "database": self.database,
+            "driver": self.driver,
+            "extraJdbc": self.extraJdbc,
+            "dbSchema": self.dbSchema,
+            "filename": self.filename,
+            "sheets": self.sheets
+        }
 
 
 class TableSchema:

backend/apps/db/db.py

@@ -8,10 +8,11 @@ from sqlalchemy.orm import sessionmaker
 
 from apps.datasource.models.datasource import DatasourceConf, CoreDatasource, TableSchema, ColumnSchema
 from apps.datasource.utils.utils import aes_decrypt
+from apps.db.engine import get_engine_config
 
 
 def get_uri(ds: CoreDatasource):
-    conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration)))
+    conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) if ds.type != "excel" else get_engine_config()
     db_url: str
     if ds.type == "mysql":
         if conf.extraJdbc is not None and conf.extraJdbc != '':
@@ -23,7 +24,7 @@ def get_uri(ds: CoreDatasource):
             db_url = f"mssql+pymssql://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}?{urllib.parse.quote(conf.extraJdbc)}"
         else:
             db_url = f"mssql+pymssql://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}"
-    elif ds.type == "pg":
+    elif ds.type == "pg" or ds.type == "excel":
         if conf.extraJdbc is not None and conf.extraJdbc != '':
             db_url = f"postgresql+psycopg2://{urllib.parse.quote(conf.username)}:{urllib.parse.quote(conf.password)}@{conf.host}:{conf.port}/{urllib.parse.quote(conf.database)}?{urllib.parse.quote(conf.extraJdbc)}"
         else:
@@ -34,7 +35,7 @@ def get_uri(ds: CoreDatasource):
 
 
 def get_engine(ds: CoreDatasource) -> Engine:
-    conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration)))
+    conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) if ds.type != "excel" else get_engine_config()
     if ds.type == "pg" and (conf.dbSchema is not None and conf.dbSchema != ""):
         engine = create_engine(get_uri(ds), connect_args={"options": f"-c search_path={conf.dbSchema}"},
                                pool_timeout=30, pool_size=20, max_overflow=10)
@@ -51,7 +52,7 @@ def get_session(ds: CoreDatasource):
 
 
 def get_tables(ds: CoreDatasource):
-    conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration)))
+    conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) if ds.type != "excel" else get_engine_config()
     session = get_session(ds)
     result: Result[Any]
     sql: str = ''
@@ -82,7 +83,7 @@ def get_tables(ds: CoreDatasource):
                         t.TABLE_TYPE IN ('BASE TABLE', 'VIEW')
                         AND t.TABLE_SCHEMA = '{conf.dbSchema}';
                     """
-        elif ds.type == "pg":
+        elif ds.type == "pg" or ds.type == "excel":
             sql = """
                     SELECT 
                         c.relname AS TABLE_NAME,
@@ -113,7 +114,7 @@ def get_tables(ds: CoreDatasource):
 
 
 def get_fields(ds: CoreDatasource, table_name: str = None):
-    conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration)))
+    conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) if ds.type != "excel" else get_engine_config()
     session = get_session(ds)
     result: Result[Any]
     sql: str = ''
@@ -149,7 +150,7 @@ def get_fields(ds: CoreDatasource, table_name: str = None):
                     """
             sql2 = f" AND C.TABLE_NAME = '{table_name}';" if table_name is not None and table_name != "" else ";"
             sql = sql1 + sql2
-        elif ds.type == "pg":
+        elif ds.type == "pg" or ds.type == "excel":
             sql1 = """
                     SELECT 
                         a.attname AS COLUMN_NAME,

backend/apps/db/engine.py

@@ -1,2 +1,68 @@
 # Author: Junjun
 # Date: 2025/5/19
+from typing import List
+
+from sqlalchemy import create_engine, text, MetaData, Table
+from sqlalchemy.orm import sessionmaker
+
+from apps.datasource.models.datasource import DatasourceConf
+
+
+def get_engine_config():
+    return DatasourceConf(username="root", password="123456", host="127.0.0.1", port=5432, database="sqlbot",
+                          dbSchema="public")
+
+
+def get_engine_uri(conf: DatasourceConf):
+    return f"postgresql+psycopg2://{conf.username}:{conf.password}@{conf.host}:{conf.port}/{conf.database}"
+
+
+def get_engine_conn():
+    conf = get_engine_config()
+    db_url = get_engine_uri(conf)
+    engine = create_engine(db_url, connect_args={"options": f"-c search_path={conf.dbSchema}"}, pool_timeout=300,
+                           pool_size=20,
+                           max_overflow=10)
+    return engine
+
+
+def get_data_engine():
+    engine = get_engine_conn()
+    session_maker = sessionmaker(bind=engine)
+    session = session_maker()
+    return session
+
+
+def create_table(session, table_name: str, fields: List[any]):
+    # field type relation
+    list = []
+    for f in fields:
+        if "object" in f["type"]:
+            f["relType"] = "text"
+        elif "int" in f["type"]:
+            f["relType"] = "bigint"
+        elif "float" in f["type"]:
+            f["relType"] = "numeric"
+        elif "datetime" in f["type"]:
+            f["relType"] = "timestamp"
+        else:
+            f["relType"] = "text"
+        list.append(f'"{f["name"]}" {f["relType"]}')
+
+    sql = f"""
+            CREATE TABLE "{table_name}" (
+                {", ".join(list)}
+            );
+            """
+    session.execute(text(sql))
+    session.commit()
+
+
+def insert_data(session, table_name: str, fields: List[any], data: List[any]):
+    engine = get_engine_conn()
+    metadata = MetaData()
+    table = Table(table_name, metadata, autoload_with=engine)
+    with engine.connect() as conn:
+        stmt = table.insert().values(data)
+        conn.execute(stmt)
+        conn.commit()

frontend/src/assets/svg/ds/Excel-ds.svg

@@ -0,0 +1,3 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M3.45675 1H17.6731C17.9005 1 18.1185 1.09031 18.2792 1.25105L21.2057 4.17752C21.3664 4.33827 21.4567 4.55628 21.4567 4.78361V22.1429C21.4567 22.6162 21.073 23 20.5996 23H3.45675C2.98336 23 2.59961 22.6162 2.59961 22.1429V1.85714C2.59961 1.38376 2.98336 1 3.45675 1ZM4.65671 20.943H19.3996V6.14286H16.6139C16.4482 6.14286 16.3139 6.00854 16.3139 5.84286V3.05728H4.65671V20.943ZM17.2505 8.32229C17.3521 8.19083 17.2584 8 17.0922 8H15.0978C15.0358 8 14.9774 8.0287 14.9395 8.07771L11.8413 11.6775L13.2496 13.5L17.2505 8.32229ZM11.8413 11.6775L11.8441 11.6811L13.2496 13.5L11.8441 15.3189L11.8413 15.3224L9.0596 18.9223C9.02173 18.9713 8.96328 19 8.90134 19H6.90685C6.74071 19 6.64701 18.8092 6.74859 18.6777L10.6551 13.6223C10.7107 13.5502 10.7107 13.4497 10.6551 13.3777L6.74863 8.32229C6.64705 8.19083 6.74075 8 6.90689 8H8.90135C8.96329 8 9.02174 8.0287 9.05961 8.07771L11.8413 11.6775ZM14.9395 18.9223L11.8413 15.3224L13.2496 13.5L17.2505 18.6777C17.3521 18.8092 17.2584 19 17.0923 19H15.0978C15.0358 19 14.9774 18.9713 14.9395 18.9223Z" fill="#34C724"/>
+</svg>

frontend/src/views/ds/form.vue

@@ -32,10 +32,12 @@
         <div v-if="form.type === 'excel'">
           <el-form-item label="File">
             <el-upload
+              :disabled="!isCreate"
               accept=".xls, .xlsx, .csv"
               :headers="headers"
               action="http://localhost:8000/api/v1/datasource/uploadExcel"
               :before-upload="beforeUpload"
+              :on-success="onSuccess"
             >
               <el-button>Upload</el-button>
               <template #tip>
@@ -111,6 +113,7 @@ const isCreate = ref(true)
 const isEditTable = ref(false)
 const checkList = ref<any>([])
 const tableList = ref<any>([])
+const excelUploadSuccess = ref(false)
 
 const token = wsCache.get('user.token')
 const headers = ref<any>({'X-SQLBOT-TOKEN': `Bearer ${token}`})
@@ -138,7 +141,9 @@ const config = ref<any>({
   password:'',
   database:'',
   extraJdbc:'',
-  dbSchema:''
+  dbSchema:'',
+  filename:'',
+  sheets: []
 })
 
 const close = () => {
@@ -148,6 +153,7 @@ const close = () => {
   isEditTable.value = false
   checkList.value = []
   tableList.value = []
+  excelUploadSuccess.value = false
 }
 
 const open = (item: any, editTable: boolean = false) => {
@@ -168,6 +174,8 @@ const open = (item: any, editTable: boolean = false) => {
       config.value.database = configuration.database
       config.value.extraJdbc = configuration.extraJdbc
       config.value.dbSchema = configuration.dbSchema
+      config.value.filename = configuration.filename
+      config.value.sheets = configuration.sheets
     }
 
     if (editTable) {
@@ -175,11 +183,16 @@ const open = (item: any, editTable: boolean = false) => {
       isEditTable.value = true
       isCreate.value = false
       // request tables and check tables
+      
       datasourceApi.tableList(item.id).then((res) => {
         checkList.value = res.map((ele:any) => {return ele.table_name})
-        datasourceApi.getTablesByConf(form.value).then((res) => {
-          tableList.value = res
-        })
+        if(item.type === 'excel') {
+          tableList.value = config.value.sheets
+        } else {
+          datasourceApi.getTablesByConf(form.value).then((res) => {
+            tableList.value = res
+          })
+        }
       })
     }
   } else {
@@ -201,7 +214,9 @@ const open = (item: any, editTable: boolean = false) => {
       password:'',
       database:'',
       extraJdbc:'',
-      dbSchema:''
+      dbSchema:'',
+      filename:'',
+      sheets: []
     }
   }
   dialogVisible.value = true
@@ -250,7 +265,9 @@ const buildConf = () => {
     password:config.value.password,
     database:config.value.database,
     extraJdbc:config.value.extraJdbc,
-    dbSchema:config.value.dbSchema
+    dbSchema:config.value.dbSchema,
+    filename:config.value.filename,
+    sheets:config.value.sheets
   }))
 }
 
@@ -270,6 +287,9 @@ const check = () => {
 const next = async(formEl: FormInstance | undefined) => {
   if (form.value.type === "excel") {
     // next, show tables 
+    if (excelUploadSuccess.value) {
+      active.value++
+    }
   } else {
     if(!formEl) return
     await formEl.validate((valid) => {
@@ -302,6 +322,14 @@ const beforeUpload = (rawFile: any) => {
   return true
 }
 
+const onSuccess = (response: any) => {
+  console.log(response)
+  config.value.filename = response.data.filename
+  config.value.sheets = response.data.sheets
+  tableList.value = response.data.sheets
+  excelUploadSuccess.value = true
+}
+
 defineExpose({ open })
 </script>
 <style lang="less" scoped>

frontend/src/views/ds/index.vue

@@ -29,6 +29,7 @@
             <MysqlDs v-if="ds.type === 'mysql'"/>
             <SQLServerDs v-else-if="ds.type === 'sqlServer'"/>
             <PgDs v-else-if="ds.type === 'pg'"/>
+            <ExcelDs v-else-if="ds.type === 'excel'"/>
           </Icon>
         </div>
         <div class="connection-details">
@@ -41,7 +42,7 @@
         <div class="connection-actions">
           <el-button class="action-btn" circle @click="getTables(ds.id, ds.name)" :icon="List" />
           <el-button class="action-btn" circle @click="editTables(ds)" :icon="CreditCard" />
-          <el-button type="primary" class="action-btn" circle @click="editDs(ds)" :icon="IconOpeEdit"/>
+          <el-button v-if="ds.type !== 'excel'" type="primary" class="action-btn" circle @click="editDs(ds)" :icon="IconOpeEdit"/>
           <el-button type="danger" class="action-btn" circle @click="deleteDs(ds)" :icon="IconOpeDelete"/>
         </div>
       </div>
@@ -56,6 +57,7 @@ import IconOpeDelete from '@/assets/svg/operate/ope-delete.svg'
 import MysqlDs from '@/assets/svg/ds/mysql-ds.svg'
 import SQLServerDs from '@/assets/svg/ds/sqlServer-ds.svg'
 import PgDs from '@/assets/svg/ds/pg-ds.svg'
+import ExcelDs from '@/assets/svg/ds/Excel-ds.svg'
 import { Search, List, CreditCard } from '@element-plus/icons-vue'
 import { ref, onMounted } from 'vue'
 import DsForm from './form.vue'