feat(datasource): Datasource table virtualization

.gitignore

@@ -6,6 +6,7 @@ node_modules/
 /playwright/.cache/
 *.lock
 *-lock.json
+components.d.ts
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

frontend/src/style.less

@@ -12,9 +12,8 @@
   -webkit-font-smoothing: antialiased;
   -moz-osx-font-smoothing: grayscale;
 
-
   /* --primary-color: var(--el-color-primary); */
-  --primary-color: #1CBA90;
+  --primary-color: #1cba90;
   --primary-light: #e8f0fe;
   --secondary-color: #5f6368;
   --danger-color: #ea4335;
@@ -30,7 +29,7 @@
   --border-radius: 8px;
   --shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
 
-  --el-color-primary: #1CBA90 !important;
+  --el-color-primary: #1cba90 !important;
 }
 
 a {
@@ -50,6 +49,12 @@ body {
   min-height: 100vh;
 }
 
+* {
+  margin: 0;
+  padding: 0;
+  box-sizing: border-box;
+}
+
 #app {
   width: 100%;
   margin: 0;
@@ -67,3 +72,40 @@ body {
     background-color: #f9f9f9;
   }
 }
+
+.ellipsis {
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+
+.list-item_primary {
+  height: 40px;
+  border-radius: 4px;
+  padding: 8px 12px;
+  cursor: pointer;
+  display: flex;
+  align-items: center;
+
+  .ed-icon + .label {
+    margin-left: 5.33px;
+  }
+
+  .label {
+    width: 85%;
+    font-size: 14px;
+    .ellipsis;
+  }
+
+  &:not(.active):hover {
+    background: rgba(31, 35, 41, 0.1);
+  }
+
+  &.active {
+    background: var(--ed-color-primary-1a, rgba(51, 112, 255, 0.1));
+    color: var(--ed-color-primary);
+  }
+}
+
+
+

frontend/src/views/ds/form.vue

@@ -2,12 +2,18 @@
   <el-dialog
     v-model="dialogVisible"
     title="Add Datasource"
-    width="500"
+    width="600"
     :destroy-on-close="true"
     :close-on-click-modal="false"
     @closed="close"
+    modal-class="add-datasource_dialog"
   >
-    <el-steps v-show="isCreate" style="max-width: 600px" :active="active" align-center>
+    <el-steps
+      v-show="isCreate"
+      style="max-width: 600px"
+      :active="active"
+      align-center
+    >
       <el-step title="Base Info" />
       <el-step title="Choose Tables" />
     </el-steps>
@@ -20,7 +26,11 @@
           <el-input v-model="form.description" :rows="2" type="textarea" />
         </el-form-item>
         <el-form-item label="Type">
-          <el-select v-model="form.type" placeholder="Select Type" :disabled="!isCreate">
+          <el-select
+            v-model="form.type"
+            placeholder="Select Type"
+            :disabled="!isCreate"
+          >
             <el-option
               v-for="item in dsType"
               :key="item.value"
@@ -75,290 +85,345 @@
           </el-form-item>
           <el-form-item label="Schema" v-if="haveSchema.includes(form.type)">
             <el-input v-model="config.dbSchema" />
-            <el-button link type="primary" :icon="Plus" v-if="false">Get Schema</el-button>
+            <el-button link type="primary" :icon="Plus" v-if="false"
+              >Get Schema</el-button
+            >
           </el-form-item>
           <span v-if="form.type === 'sqlServer'">Supported version: 2012+</span>
           <span v-else-if="form.type === 'oracle'">Supported version: 12+</span>
         </div>
       </el-form>
     </div>
-    <div v-show="active === 2" class="container">
-      <el-scrollbar>
-        <el-checkbox-group v-model="checkList">
-          <el-row :gutter="10">
-            <el-col v-for="item in tableList" :key="item.value" :span="12">
-              <el-checkbox :label="item.tableName">{{ item.tableName }}</el-checkbox>
-            </el-col>
-          </el-row>
-        </el-checkbox-group>
-      </el-scrollbar>
+    <div v-show="active === 2" class="container" v-loading="tableListLoading">
+      <el-checkbox-group v-model="checkList" style="position: relative">
+        <FixedSizeList
+          :itemSize="40"
+          :data="tableList"
+          :total="tableList.length"
+          :width="560"
+          :height="400"
+          :scrollbarAlwaysOn="true"
+          class-name="el-select-dropdown__list"
+          layout="vertical"
+        >
+          <template #default="{ index, style }">
+            <div class="list-item_primary" :style="style">
+              <el-checkbox :label="tableList[index].tableName">{{
+                tableList[index].tableName
+              }}</el-checkbox>
+            </div>
+          </template>
+        </FixedSizeList>
+      </el-checkbox-group>
       <span>Selected: {{ checkList.length }}/{{ tableList.length }}</span>
     </div>
-    <div style="display: flex;justify-content: flex-end;margin-top: 20px;">
+    <div style="display: flex; justify-content: flex-end; margin-top: 20px">
       <el-button @click="close">Cancel</el-button>
-      <el-button v-show="!isCreate && !isEditTable && form.type !== 'excel'" @click="check">Test Connect</el-button>
-      <el-button v-show="active === 1 && isCreate" type="primary" @click="next(dsFormRef)">Next</el-button>
-      <el-button v-show="active === 2 && isCreate" @click="preview">Preview</el-button>
-      <el-button v-show="active === 2 || !isCreate" type="primary" @click="save(dsFormRef)">Save</el-button>
+      <el-button
+        v-show="!isCreate && !isEditTable && form.type !== 'excel'"
+        @click="check"
+        >Test Connect</el-button
+      >
+      <el-button
+        v-show="active === 1 && isCreate"
+        type="primary"
+        @click="next(dsFormRef)"
+        >Next</el-button
+      >
+      <el-button v-show="active === 2 && isCreate" @click="preview"
+        >Preview</el-button
+      >
+      <el-button
+        v-show="active === 2 || !isCreate"
+        type="primary"
+        @click="save(dsFormRef)"
+        >Save</el-button
+      >
     </div>
   </el-dialog>
 </template>
 <script lang="ts" setup>
-import { ref, reactive } from 'vue'
-import { datasourceApi } from '@/api/datasource'
-import { encrypted, decrypted } from './js/aes'
-import { ElMessage } from 'element-plus'
-import type { FormInstance, FormRules } from 'element-plus'
-import { Plus } from '@element-plus/icons-vue'
-import { useCache } from '@/utils/useCache'
-import { dsType, haveSchema} from '@/views/ds/js/ds-type'
-
-const { wsCache } = useCache()
-const dsFormRef = ref<FormInstance>()
-const emit = defineEmits(['refresh'])
-const active = ref(1)
-const isCreate = ref(true)
-const isEditTable = ref(false)
-const checkList = ref<any>([])
-const tableList = ref<any>([])
-const excelUploadSuccess = ref(false)
-
-const token = wsCache.get('user.token')
-const headers = ref<any>({'X-SQLBOT-TOKEN': `Bearer ${token}`})
+import { ref, reactive } from "vue";
+import { datasourceApi } from "@/api/datasource";
+import { encrypted, decrypted } from "./js/aes";
+import { ElMessage } from "element-plus";
+import type { FormInstance, FormRules } from "element-plus";
+import FixedSizeList from "element-plus/es/components/virtual-list/src/components/fixed-size-list.mjs";
+import { Plus } from "@element-plus/icons-vue";
+import { useCache } from "@/utils/useCache";
+import { dsType, haveSchema } from "@/views/ds/js/ds-type";
 
+const { wsCache } = useCache();
+const dsFormRef = ref<FormInstance>();
+const emit = defineEmits(["refresh"]);
+const active = ref(1);
+const isCreate = ref(true);
+const isEditTable = ref(false);
+const checkList = ref<any>([]);
+const tableList = ref<any>([]);
+const excelUploadSuccess = ref(false);
+const tableListLoading = ref(true);
+const token = wsCache.get("user.token");
+const headers = ref<any>({ "X-SQLBOT-TOKEN": `Bearer ${token}` });
 
 const rules = reactive<FormRules>({
   name: [
-    { required: true, message: 'Please input name', trigger: 'blur' },
-    { min: 1, max: 50, message: 'Length should be 1 to 50', trigger: 'blur' },
+    { required: true, message: "Please input name", trigger: "blur" },
+    { min: 1, max: 50, message: "Length should be 1 to 50", trigger: "blur" },
   ],
-})
+});
 
-const dialogVisible = ref<boolean>(false)
+const dialogVisible = ref<boolean>(false);
 const form = ref<any>({
-  name:'',
-  description:'',
-  type:'mysql',
-  configuration: ''
-})
+  name: "",
+  description: "",
+  type: "mysql",
+  configuration: "",
+});
 const config = ref<any>({
-  driver:'',
-  host:'',
-  port:0,
-  username:'',
-  password:'',
-  database:'',
-  extraJdbc:'',
-  dbSchema:'',
-  filename:'',
+  driver: "",
+  host: "",
+  port: 0,
+  username: "",
+  password: "",
+  database: "",
+  extraJdbc: "",
+  dbSchema: "",
+  filename: "",
   sheets: [],
-  mode:'service_name'
-})
+  mode: "service_name",
+});
 
 const close = () => {
-  dialogVisible.value = false
-  isCreate.value = true
-  active.value = 1
-  isEditTable.value = false
-  checkList.value = []
-  tableList.value = []
-  excelUploadSuccess.value = false
-}
+  dialogVisible.value = false;
+  isCreate.value = true;
+  active.value = 1;
+  isEditTable.value = false;
+  checkList.value = [];
+  tableList.value = [];
+  excelUploadSuccess.value = false;
+};
 
 const open = (item: any, editTable: boolean = false) => {
-  isEditTable.value = false
+  isEditTable.value = false;
   if (item) {
-    isCreate.value = false
-    form.value.id = item.id
-    form.value.name = item.name
-    form.value.description = item.description
-    form.value.type = item.type
-    form.value.configuration = item.configuration
-    if(item.configuration) {
-      const configuration = JSON.parse(decrypted(item.configuration))
-      config.value.host = configuration.host
-      config.value.port = configuration.port
-      config.value.username = configuration.username
-      config.value.password = configuration.password
-      config.value.database = configuration.database
-      config.value.extraJdbc = configuration.extraJdbc
-      config.value.dbSchema = configuration.dbSchema
-      config.value.filename = configuration.filename
-      config.value.sheets = configuration.sheets
-      config.value.mode = configuration.mode
+    isCreate.value = false;
+    form.value.id = item.id;
+    form.value.name = item.name;
+    form.value.description = item.description;
+    form.value.type = item.type;
+    form.value.configuration = item.configuration;
+    if (item.configuration) {
+      const configuration = JSON.parse(decrypted(item.configuration));
+      config.value.host = configuration.host;
+      config.value.port = configuration.port;
+      config.value.username = configuration.username;
+      config.value.password = configuration.password;
+      config.value.database = configuration.database;
+      config.value.extraJdbc = configuration.extraJdbc;
+      config.value.dbSchema = configuration.dbSchema;
+      config.value.filename = configuration.filename;
+      config.value.sheets = configuration.sheets;
+      config.value.mode = configuration.mode;
     }
 
     if (editTable) {
-      active.value = 2
-      isEditTable.value = true
-      isCreate.value = false
+      active.value = 2;
+      isEditTable.value = true;
+      isCreate.value = false;
       // request tables and check tables
-      
+
       datasourceApi.tableList(item.id).then((res) => {
-        checkList.value = res.map((ele:any) => {return ele.table_name})
-        if(item.type === 'excel') {
-          tableList.value = config.value.sheets
+        checkList.value = res.map((ele: any) => {
+          return ele.table_name;
+        });
+        if (item.type === "excel") {
+          tableList.value = config.value.sheets;
         } else {
-          datasourceApi.getTablesByConf(form.value).then((table) => {
-            tableList.value = table
-            checkList.value = checkList.value.filter((ele:string) => {return table.map((ele:any) => {return ele.tableName}).includes(ele)})
-          })
+          tableListLoading.value = true;
+          datasourceApi
+            .getTablesByConf(form.value)
+            .then((table) => {
+              tableList.value = table;
+              checkList.value = checkList.value.filter((ele: string) => {
+                return table
+                  .map((ele: any) => {
+                    return ele.tableName;
+                  })
+                  .includes(ele);
+              });
+            })
+            .finally(() => {
+              tableListLoading.value = false;
+            });
         }
-      })
+      });
     }
   } else {
-    isCreate.value = true
-    isEditTable.value = false
-    checkList.value = []
-    tableList.value = []
+    isCreate.value = true;
+    isEditTable.value = false;
+    checkList.value = [];
+    tableList.value = [];
     form.value = {
-      name:'',
-      description:'',
-      type:'mysql',
-      configuration: ''
-    }
+      name: "",
+      description: "",
+      type: "mysql",
+      configuration: "",
+    };
     config.value = {
-      driver:'',
-      host:'',
-      port:0,
-      username:'',
-      password:'',
-      database:'',
-      extraJdbc:'',
-      dbSchema:'',
-      filename:'',
+      driver: "",
+      host: "",
+      port: 0,
+      username: "",
+      password: "",
+      database: "",
+      extraJdbc: "",
+      dbSchema: "",
+      filename: "",
       sheets: [],
-      mode:'service_name'
-    }
+      mode: "service_name",
+    };
   }
-  dialogVisible.value = true
-}
+  dialogVisible.value = true;
+};
 
-const save = async(formEl: FormInstance | undefined) => {
-  if(!formEl) return
+const save = async (formEl: FormInstance | undefined) => {
+  if (!formEl) return;
   await formEl.validate((valid) => {
     if (valid) {
       const list = tableList.value
-          .filter((ele: any) => {return checkList.value.includes(ele.tableName)})
-          .map((ele: any) => {return {"table_name": ele.tableName, "table_comment": ele.tableComment}})
+        .filter((ele: any) => {
+          return checkList.value.includes(ele.tableName);
+        })
+        .map((ele: any) => {
+          return { table_name: ele.tableName, table_comment: ele.tableComment };
+        });
 
-      buildConf()
+      buildConf();
       if (form.value.id) {
         if (!isEditTable.value) {
           // only update datasource config info
           datasourceApi.update(form.value).then((res) => {
-            console.log(res)
-            close()
-            emit('refresh')
-          })
+            console.log(res);
+            close();
+            emit("refresh");
+          });
         } else {
           // save table and field
           datasourceApi.chooseTables(form.value.id, list).then(() => {
-            close()
-          })
+            close();
+          });
         }
       } else {
-        form.value.tables = list
+        form.value.tables = list;
         datasourceApi.add(form.value).then((res: any) => {
-          console.log(res)
-          close()
-          emit('refresh')
-        })
+          console.log(res);
+          close();
+          emit("refresh");
+        });
       }
     }
-  })
-}
+  });
+};
 
 const buildConf = () => {
-  form.value.configuration = encrypted(JSON.stringify({
-    host:config.value.host,
-    port:config.value.port,
-    username:config.value.username,
-    password:config.value.password,
-    database:config.value.database,
-    extraJdbc:config.value.extraJdbc,
-    dbSchema:config.value.dbSchema,
-    filename:config.value.filename,
-    sheets:config.value.sheets,
-    mode:config.value.mode
-  }))
-}
+  form.value.configuration = encrypted(
+    JSON.stringify({
+      host: config.value.host,
+      port: config.value.port,
+      username: config.value.username,
+      password: config.value.password,
+      database: config.value.database,
+      extraJdbc: config.value.extraJdbc,
+      dbSchema: config.value.dbSchema,
+      filename: config.value.filename,
+      sheets: config.value.sheets,
+      mode: config.value.mode,
+    })
+  );
+};
 
 const check = () => {
-  buildConf()
+  buildConf();
   datasourceApi.check(form.value).then((res: any) => {
-    if(res) {
+    if (res) {
       ElMessage({
-        message: 'Connect success',
-        type: 'success',
-        showClose: true
-      })
+        message: "Connect success",
+        type: "success",
+        showClose: true,
+      });
     } else {
       ElMessage({
-        message: 'Connect failed',
-        type: 'error',
-        showClose: true
-      })
+        message: "Connect failed",
+        type: "error",
+        showClose: true,
+      });
     }
-  })
-}
+  });
+};
 
-const next = async(formEl: FormInstance | undefined) => {
-  if(!formEl) return
+const next = async (formEl: FormInstance | undefined) => {
+  if (!formEl) return;
   await formEl.validate((valid) => {
     if (valid) {
       if (form.value.type === "excel") {
-        // next, show tables 
+        // next, show tables
         if (excelUploadSuccess.value) {
-          active.value++
+          active.value++;
         }
       } else {
         // check status if success do next
-        buildConf()
+        buildConf();
         datasourceApi.check(form.value).then((res: boolean) => {
-          if(res) {
-            active.value++
+          if (res) {
+            active.value++;
             // request tables
             datasourceApi.getTablesByConf(form.value).then((res) => {
-              tableList.value = res
-            })
+              tableList.value = res;
+            });
           } else {
             ElMessage({
-              message: 'Connect failed',
-              type: 'error',
-              showClose: true
-            })
+              message: "Connect failed",
+              type: "error",
+              showClose: true,
+            });
           }
-        })
+        });
       }
     }
-  })
-}
+  });
+};
 
 const preview = () => {
-  active.value--
-}
+  active.value--;
+};
 
 const beforeUpload = (rawFile: any) => {
   if (rawFile.size / 1024 / 1024 > 50) {
-    ElMessage.error('File size can not exceed 50MB!')
-    return false
+    ElMessage.error("File size can not exceed 50MB!");
+    return false;
   }
-  return true
-}
+  return true;
+};
 
 const onSuccess = (response: any) => {
-  config.value.filename = response.data.filename
-  config.value.sheets = response.data.sheets
-  tableList.value = response.data.sheets
-  excelUploadSuccess.value = true
-}
+  config.value.filename = response.data.filename;
+  config.value.sheets = response.data.sheets;
+  tableList.value = response.data.sheets;
+  excelUploadSuccess.value = true;
+};
 
-defineExpose({ open })
+defineExpose({ open });
 </script>
-<style lang="less" scoped>
-.container{
-  height: 600px;
-  margin-top: 20px;
+<style lang="less">
+.add-datasource_dialog {
+  .container {
+    max-height: 600px;
+    margin-top: 20px;
+    .el-vl__window.el-select-dropdown__list::-webkit-scrollbar {
+      width: 0;
+      height: 0;
+    }
+  }
 }
-</style>
+</style>

frontend/src/views/ds/index.vue

@@ -180,7 +180,6 @@ onMounted(() => {
     position: relative;
     border: 1px solid #dadce0;
     transition: all 0.2s ease;
-    height: 110px;
     align-items: center;
   }
   .connection-icon {