Merge remote-tracking branch 'origin/main'

backend/apps/datasource/api/datasource.py

@@ -1,7 +1,10 @@
 from fastapi import APIRouter
-from ..crud.datasource import get_datasource_list, check_status, create_ds, update_ds, delete_ds, getTables, getFields, execSql
+from ..crud.datasource import get_datasource_list, check_status, create_ds, update_ds, delete_ds, getTables, getFields, \
+    execSql, update_table_and_fields, getTablesByDs
 from common.core.deps import SessionDep
-from ..models.datasource import CoreDatasource, CreateDatasource
+from ..models.datasource import CoreDatasource, CreateDatasource, EditObj
+from ..crud.table import get_tables_by_ds_id
+from ..crud.field import get_fields_by_table_id
 
 router = APIRouter(tags=["datasource"], prefix="/datasource")
 
@@ -36,6 +39,11 @@ async def get_tables(session: SessionDep, id: int):
     return getTables(session, id)
 
 
+@router.post("/getTablesByConf")
+async def get_tables_by_conf(session: SessionDep, ds: CoreDatasource):
+    return getTablesByDs(session, ds)
+
+
 @router.post("/getFields/{id}/{table_name}")
 async def get_tables(session: SessionDep, id: int, table_name: str):
     return getFields(session, id, table_name)
@@ -44,3 +52,18 @@ async def get_tables(session: SessionDep, id: int, table_name: str):
 @router.post("/execSql/{id}/{sql}")
 async def exec_sql(session: SessionDep, id: int, sql: str):
     return execSql(session, id, sql)
+
+
+@router.get("/tableList/{id}")
+async def table_list(session: SessionDep, id: int):
+    return get_tables_by_ds_id(session, id)
+
+
+@router.get("/fieldList/{id}")
+async def field_list(session: SessionDep, id: int):
+    return get_fields_by_table_id(session, id)
+
+
+@router.post("/editLocalComment")
+async def edit_local(session: SessionDep, data: EditObj):
+    update_table_and_fields(session, data)

backend/apps/datasource/crud/datasource.py

@@ -1,13 +1,13 @@
 from sqlmodel import select
-from ..models.datasource import CoreDatasource, CreateDatasource, CoreTable, CoreField, ColumnSchema
+from ..models.datasource import CoreDatasource, CreateDatasource, CoreTable, CoreField, ColumnSchema, EditObj
 import datetime
 from common.core.deps import SessionDep
 from apps.db.db import get_session, get_tables, get_fields, exec_sql
 from sqlalchemy import text, and_
 from common.utils.utils import deepcopy_ignore_extra
 from typing import List
-from ..crud.table import delete_table_by_ds_id
-from ..crud.field import delete_field_by_ds_id
+from ..crud.table import delete_table_by_ds_id, update_table
+from ..crud.field import delete_field_by_ds_id, update_field
 
 
 def get_datasource_list(session: SessionDep) -> CoreDatasource:
@@ -80,6 +80,11 @@ def getTables(session: SessionDep, id: int):
     return tables
 
 
+def getTablesByDs(session: SessionDep, ds: CoreDatasource):
+    tables = get_tables(ds)
+    return tables
+
+
 def getFields(session: SessionDep, id: int, table_name: str):
     ds = session.exec(select(CoreDatasource).where(CoreDatasource.id == id)).first()
     fields = get_fields(ds, table_name)
@@ -151,3 +156,9 @@ def sync_fields(session: SessionDep, ds: CoreDatasource, table: CoreTable, field
     if len(id_list) > 0:
         session.query(CoreField).filter(and_(CoreField.table_id == table.id, CoreField.id.not_in(id_list))).delete(
             synchronize_session=False)
+
+
+def update_table_and_fields(session: SessionDep, data: EditObj):
+    update_table(session, data.table)
+    for field in data.fields:
+        update_field(session, field)

backend/apps/datasource/crud/field.py

@@ -1,7 +1,18 @@
 from common.core.deps import SessionDep
 from ..models.datasource import CoreDatasource, CreateDatasource, CoreTable, CoreField, ColumnSchema
+from sqlalchemy import and_
 
 
 def delete_field_by_ds_id(session: SessionDep, id: int):
     session.query(CoreField).filter(CoreField.ds_id == id).delete(synchronize_session=False)
     session.commit()
+
+
+def get_fields_by_table_id(session: SessionDep, id: int):
+    return session.query(CoreField).filter(CoreField.table_id == id).order_by(
+        CoreField.field_name.asc()).all()
+
+
+def update_field(session: SessionDep, item: CoreField):
+    session.add(item)
+    session.commit()

backend/apps/datasource/crud/table.py

@@ -1,7 +1,18 @@
 from common.core.deps import SessionDep
 from ..models.datasource import CoreDatasource, CreateDatasource, CoreTable, CoreField, ColumnSchema
+from sqlalchemy import and_
 
 
 def delete_table_by_ds_id(session: SessionDep, id: int):
     session.query(CoreTable).filter(CoreTable.ds_id == id).delete(synchronize_session=False)
     session.commit()
+
+
+def get_tables_by_ds_id(session: SessionDep, id: int):
+    return session.query(CoreTable).filter(CoreTable.ds_id == id).order_by(
+        CoreTable.table_name.asc()).all()
+
+
+def update_table(session: SessionDep, item: CoreTable):
+    session.add(item)
+    session.commit()

backend/apps/datasource/models/datasource.py

@@ -20,7 +20,7 @@ class CoreDatasource(SQLModel, table=True):
 class CoreTable(SQLModel, table=True):
     __tablename__ = "core_table"
     id: int = Field(sa_column=Column(Integer, Identity(always=True), nullable=False, primary_key=True))
-    ds_id:int = Field(sa_column=Column(BigInteger()))
+    ds_id: int = Field(sa_column=Column(BigInteger()))
     checked: bool = Field(default=True)
     table_name: str = Field(sa_column=Column(Text))
     table_comment: str = Field(sa_column=Column(Text))
@@ -52,6 +52,13 @@ class CreateDatasource(BaseModel):
     tables: List[CoreTable] = []
 
 
+# edit local saved table and fields
+class EditObj(BaseModel):
+    table: CoreTable = None
+    fields: List[CoreField] = []
+
+
+# datasource config info
 class DatasourceConf(BaseModel):
     host: str = ''
     port: int = 0

frontend/src/api/datasource.ts

@@ -7,6 +7,7 @@ export const datasourceApi = {
   update: (data: any) => request.post('/datasource/update', data),
   delete: (id: Number) => request.post(`/datasource/delete/${id}`),
   getTables: (id: Number) => request.post(`/datasource/getTables/${id}`),
+  getTablesByConf: (data: any) => request.post('/datasource/getTablesByConf', data),
   getFields: (id: Number, table_name: string) => request.post(`/datasource/getFields/${id}/${table_name}`),
   execSql: (id: Number, sql: string) => request.post(`/datasource/execSql/${id}/${sql}`),
 }

frontend/src/style.less

@@ -50,19 +50,9 @@ body {
   min-height: 100vh;
 }
 
-h1 {
-  font-size: 3.2em;
-  line-height: 1.1;
-}
-
-.card {
-  padding: 2em;
-}
-
 #app {
   width: 100%;
   margin: 0;
-  text-align: center;
 }
 
 @media (prefers-color-scheme: light) {

frontend/src/views/ds/form.vue

@@ -4,46 +4,58 @@
     title="Add Datasource"
     width="500"
     :destroy-on-close="true"
+    :close-on-click-modal="false"
+    @closed="close"
   >
-    <el-form :model="form" label-width="auto" ref="dsFormRef" :rules="rules">
-      <el-form-item label="Name" prop="name">
-        <el-input v-model="form.name" />
-      </el-form-item>
-      <el-form-item label="Description">
-        <el-input v-model="form.description" :rows="2" type="textarea" />
-      </el-form-item>
-      <el-form-item label="Type">
-        <el-select v-model="form.type" placeholder="Select Type">
-          <el-option
-            v-for="item in dsType"
-            :key="item.value"
-            :label="item.label"
-            :value="item.value"
-          />
-        </el-select>
-      </el-form-item>
-      <el-form-item label="Host/Ip">
-        <el-input v-model="config.host" />
-      </el-form-item>
-      <el-form-item label="Port">
-        <el-input v-model="config.port" />
-      </el-form-item>
-      <el-form-item label="Username">
-        <el-input v-model="config.username" />
-      </el-form-item>
-      <el-form-item label="Password">
-        <el-input v-model="config.password" type="password" show-password />
-      </el-form-item>
-      <el-form-item label="Database">
-        <el-input v-model="config.database" />
-      </el-form-item>
-
-      <div style="display: flex;justify-content: flex-end;">
-        <el-button @click="close">Cancel</el-button>
-        <el-button @click="check">Test Connect</el-button>
-        <el-button type="primary" @click="save(dsFormRef)">Save</el-button>
-      </div>
-    </el-form>
+    <el-steps v-show="isCreate" style="max-width: 600px" :active="active" align-center>
+      <el-step title="Base Info" />
+      <el-step title="Choose Tables" />
+    </el-steps>
+    <div v-show="active === 1" class="container">
+      <el-form :model="form" label-width="auto" ref="dsFormRef" :rules="rules">
+        <el-form-item label="Name" prop="name">
+          <el-input v-model="form.name" />
+        </el-form-item>
+        <el-form-item label="Description">
+          <el-input v-model="form.description" :rows="2" type="textarea" />
+        </el-form-item>
+        <el-form-item label="Type">
+          <el-select v-model="form.type" placeholder="Select Type">
+            <el-option
+              v-for="item in dsType"
+              :key="item.value"
+              :label="item.label"
+              :value="item.value"
+            />
+          </el-select>
+        </el-form-item>
+        <el-form-item label="Host/Ip">
+          <el-input v-model="config.host" />
+        </el-form-item>
+        <el-form-item label="Port">
+          <el-input v-model="config.port" />
+        </el-form-item>
+        <el-form-item label="Username">
+          <el-input v-model="config.username" />
+        </el-form-item>
+        <el-form-item label="Password">
+          <el-input v-model="config.password" type="password" show-password />
+        </el-form-item>
+        <el-form-item label="Database">
+          <el-input v-model="config.database" />
+        </el-form-item>
+      </el-form>
+    </div>
+    <div v-show="active === 2" class="container">
+      222
+    </div>
+    <div style="display: flex;justify-content: flex-end;">
+      <el-button @click="close">Cancel</el-button>
+      <el-button v-show="!isCreate && !isEditTable" @click="check">Test Connect</el-button>
+      <el-button v-show="active === 1 && isCreate" type="primary" @click="next">Next</el-button>
+      <el-button v-show="active === 2 && isCreate" @click="preview">Preview</el-button>
+      <el-button v-show="active === 2 || !isCreate" type="primary" @click="save(dsFormRef)">Save</el-button>
+    </div>
   </el-dialog>
 </template>
 <script lang="ts" setup>
@@ -55,6 +67,9 @@ import type { FormInstance, FormRules } from 'element-plus'
 
 const dsFormRef = ref<FormInstance>()
 const emit = defineEmits(['refresh'])
+const active = ref(1)
+const isCreate = ref(true)
+const isEditTable = ref(false)
 
 const rules = reactive<FormRules>({
   name: [
@@ -84,36 +99,48 @@ const config = ref<any>({
 
 const close = () => {
   dialogVisible.value = false
+  isCreate.value = true
+  active.value = 1
+  isEditTable.value = false
 }
 
-const open = (item: any) => {
-  if (item) {
-    form.value.id = item.id
-    form.value.name = item.name
-    form.value.description = item.description
-    form.value.type = item.type
-    if(item.configuration) {
-      const configuration = JSON.parse(decrypted(item.configuration))
-      config.value.host = configuration.host
-      config.value.port = configuration.port
-      config.value.username = configuration.username
-      config.value.password = configuration.password
-      config.value.database = configuration.database
-    }
+const open = (item: any, editTable: boolean = false) => {
+  if (editTable) {
+    active.value = 2
+    isEditTable.value = true
+    isCreate.value = false
   } else {
-    form.value = {
-      name:'',
-      description:'',
-      type:'mysql',
-      configuration: ''
-    }
-    config.value = {
-      driver:'',
-      host:'',
-      port:0,
-      username:'',
-      password:'',
-      database:'',
+    isEditTable.value = false
+    if (item) {
+      isCreate.value = false
+      form.value.id = item.id
+      form.value.name = item.name
+      form.value.description = item.description
+      form.value.type = item.type
+      if(item.configuration) {
+        const configuration = JSON.parse(decrypted(item.configuration))
+        config.value.host = configuration.host
+        config.value.port = configuration.port
+        config.value.username = configuration.username
+        config.value.password = configuration.password
+        config.value.database = configuration.database
+      }
+    } else {
+      isCreate.value = true
+      form.value = {
+        name:'',
+        description:'',
+        type:'mysql',
+        configuration: ''
+      }
+      config.value = {
+        driver:'',
+        host:'',
+        port:0,
+        username:'',
+        password:'',
+        database:'',
+      }
     }
   }
   dialogVisible.value = true
@@ -131,13 +158,18 @@ const save = async(formEl: FormInstance | undefined) => {
         database:config.value.database
       }))
       if (form.value.id) {
-        datasourceApi.update(form.value).then((res) => {
-          console.log(res)
-          close()
-          emit('refresh')
-        })
+        if (!isEditTable.value) {
+          // only update datasource config info
+          datasourceApi.update(form.value).then((res) => {
+            console.log(res)
+            close()
+            emit('refresh')
+          })
+        } else {
+          // save table and field
+        }
       } else {
-        form.value.tables = [{"table_name":"core_dataset_table"},{"table_name":"pre_user"}]
+        form.value.tables = [{"table_name":"core_dataset_table"},{"table_name":"pre_user"}]// todo test
         datasourceApi.add(form.value).then((res: any) => {
           console.log(res)
           close()
@@ -148,7 +180,7 @@ const save = async(formEl: FormInstance | undefined) => {
   })
 }
 
-const check = () => {
+const buildConf = () => {
   form.value.configuration = encrypted(JSON.stringify({
     host:config.value.host,
     port:config.value.port,
@@ -156,6 +188,10 @@ const check = () => {
     password:config.value.password,
     database:config.value.database
   }))
+}
+
+const check = () => {
+  buildConf()
   datasourceApi.check(form.value).then((res: any) => {
     console.log(res)
     if(res) {
@@ -168,7 +204,30 @@ const check = () => {
   })
 }
 
+const next = () => {
+  // check status if success do next
+  buildConf()
+  datasourceApi.check(form.value).then((res: any) => {
+    console.log(res)
+    if(res) {
+      active.value++
+      // request tables
+      datasourceApi.getTablesByConf(form.value).then((res) => {
+        console.log(res)
+      })
+    }
+  })
+}
+
+const preview = () => {
+  active.value--
+}
+
 defineExpose({ open })
 </script>
 <style lang="less" scoped>
+.container{
+  height: 500px;
+  margin-top: 20px;
+}
 </style>