feat(Data Source): Supports batch uploading of data tables (Note)

frontend/src/api/datasource.ts

@@ -27,4 +27,9 @@ export const datasourceApi = {
   cancelRequests: () => request.cancelRequests(),
   getSchema: (data: any) => request.post('/datasource/getSchemaByConf', data),
   syncFields: (id: number) => request.post(`/datasource/syncFields/${id}`),
+  exportDsSchema: (id: any) =>
+    request.get(`/datasource/exportDsSchema/${id}`, {
+      responseType: 'blob',
+      requestOptions: { customError: true },
+    }),
 }

frontend/src/assets/svg/icon_import_outlined.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M8.1153 8.65984L9.55944 10.0713C9.69262 10.2015 9.69262 10.4125 9.55944 10.5427L9.07712 11.0141C8.94393 11.1443 8.72799 11.1443 8.5948 11.0141L6.42437 8.89279L5.94206 8.42139C5.67568 8.16104 5.67568 7.73893 5.94206 7.47858L8.5948 4.88585C8.72799 4.75568 8.94393 4.75568 9.07712 4.88585L9.55944 5.35726C9.69262 5.48743 9.69262 5.69849 9.55944 5.82866L7.99015 7.36244L14.3241 7.33506C14.3251 7.33505 14.3261 7.33505 14.327 7.33505C14.5154 7.33582 14.6674 7.48567 14.6667 7.66977L14.664 8.30105C14.6632 8.48403 14.5117 8.63219 14.3244 8.633L8.1153 8.65984ZM10.5417 2.66665H2.69753V13.3333H10.5417C10.73 13.3333 10.8827 13.4826 10.8827 13.6666V14.3333C10.8827 14.5174 10.73 14.6666 10.5417 14.6666H2.01543C1.63871 14.6666 1.33333 14.3682 1.33333 14V1.99998C1.33333 1.63179 1.63871 1.33331 2.01543 1.33331H10.5417C10.73 1.33331 10.8827 1.48255 10.8827 1.66665V2.33331C10.8827 2.51741 10.73 2.66665 10.5417 2.66665Z" fill="#1F2329"/>
+</svg>

frontend/src/i18n/en.json

@@ -8,6 +8,7 @@
     "Details": "Details"
   },
   "parameter": {
+    "memo": "Memo update rules: Match table name and field name. If a match is found, update the table memo and field memo; otherwise, leave the memo unchanged.",
     "parameter_configuration": "Parameter Config",
     "question_count_settings": "Question Count Settings",
     "model_thinking_process": "Expand Model Thinking Process",

frontend/src/i18n/ko-KR.json

@@ -8,6 +8,7 @@
     "Details": "세부"
   },
   "parameter": {
+    "memo": "메모 업데이트 규칙: 테이블 이름과 필드 이름이 일치하는지 확인합니다. 일치하는 항목이 있으면 테이블 메모와 필드 메모를 업데이트하고, 그렇지 않으면 메모를 변경하지 않고 그대로 둡니다.",
     "parameter_configuration": "매개변수 구성",
     "question_count_settings": "질문 수 설정",
     "model_thinking_process": "모델 사고 프로세스 확장",
@@ -819,4 +820,4 @@
   "modelType": {
     "llm": "대형 언어 모델"
   }
-}
+}

frontend/src/i18n/zh-CN.json

@@ -8,6 +8,7 @@
     "Details": "详情"
   },
   "parameter": {
+    "memo": "备注更新规则：根据表名和字段名匹配，如果匹配则更新表备注和字段备注；如果匹配不上，备注保持不变。",
     "parameter_configuration": "参数配置",
     "question_count_settings": "问数设置",
     "model_thinking_process": "展开模型思考过程",
@@ -140,7 +141,7 @@
     "click_to_select_file": "点击选择文件",
     "upload_hint_first": "先",
     "upload_hint_download_template": "下载模板",
-    "upload_hint_end": "，按要求填写后上传",
+    "upload_hint_end": "，按要求填写后上传。",
     "continue_to_upload": "继续导入",
     "reset_password": "重置密码",
     "password_reset_successful": "重置密码成功",

frontend/src/views/ds/DataTable.vue

@@ -3,14 +3,17 @@ import { ref, computed, onMounted, reactive } from 'vue'
 import { datasourceApi } from '@/api/datasource'
 import icon_right_outlined from '@/assets/svg/icon_right_outlined.svg'
 import icon_form_outlined from '@/assets/svg/icon_form_outlined.svg'
+import icon_import_outlined from '@/assets/svg/icon_import_outlined.svg'
 import icon_searchOutline_outlined from '@/assets/svg/icon_search-outline_outlined.svg'
 import EmptyBackground from '@/views/dashboard/common/EmptyBackground.vue'
 import edit from '@/assets/svg/icon_edit_outlined.svg'
 import { useI18n } from 'vue-i18n'
 import ParamsForm from './ParamsForm.vue'
+import UploaderRemark from '@/views/system/excel-upload/UploaderRemark.vue'
 import TableRelationship from '@/views/ds/TableRelationship.vue'
 import icon_mindnote_outlined from '@/assets/svg/icon_mindnote_outlined.svg'
 import { Refresh } from '@element-plus/icons-vue'
+
 interface Table {
   name: string
   host: string
@@ -106,17 +109,29 @@ const fieldListComputed = computed(() => {
   return fieldList.value.slice((currentPage - 1) * pageSize, currentPage * pageSize)
 })
 
-const init = () => {
+const init = (reset = false) => {
   initLoading.value = true
   datasourceApi.getDs(props.info.id).then((res) => {
     ds.value = res
     fieldList.value = []
     pageInfo.total = 0
     pageInfo.currentPage = 1
-    datasourceApi.tableList(props.info.id).then((res) => {
-      tableList.value = res
-      initLoading.value = false
-    })
+    datasourceApi
+      .tableList(props.info.id)
+      .then((res) => {
+        tableList.value = res
+
+        if (currentTable.value?.id && reset) {
+          tableList.value.forEach((ele) => {
+            if (ele.id === currentTable.value?.id) {
+              clickTable(ele)
+            }
+          })
+        }
+      })
+      .finally(() => {
+        initLoading.value = false
+      })
   })
 }
 onMounted(() => {
@@ -237,6 +252,47 @@ const syncFields = () => {
     })
 }
 
+function downloadTemplate() {
+  datasourceApi
+    .exportDsSchema(props.info.id)
+    .then((res) => {
+      const blob = new Blob([res], {
+        type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+      })
+      const link = document.createElement('a')
+      link.href = URL.createObjectURL(blob)
+      link.download = props.info.name + '.xlsx'
+      document.body.appendChild(link)
+      link.click()
+      document.body.removeChild(link)
+    })
+    .catch(async (error) => {
+      if (error.response) {
+        try {
+          let text = await error.response.data.text()
+          try {
+            text = JSON.parse(text)
+          } finally {
+            ElMessage({
+              message: text,
+              type: 'error',
+              showClose: true,
+            })
+          }
+        } catch (e) {
+          console.error('Error processing error response:', e)
+        }
+      } else {
+        console.error('Other error:', error)
+        ElMessage({
+          message: error,
+          type: 'error',
+          showClose: true,
+        })
+      }
+    })
+}
+
 const emits = defineEmits(['back', 'refresh'])
 const back = () => {
   emits('back')
@@ -298,6 +354,16 @@ const btnSelectClick = (val: any) => {
         <icon_right_outlined></icon_right_outlined>
       </el-icon>
       <div class="name">{{ info.name }}</div>
+      <el-button @click="downloadTemplate" class="export-remark" secondary>
+        <template #icon>
+          <icon_import_outlined></icon_import_outlined>
+        </template>
+        {{ $t('professional.export') }}
+      </el-button>
+      <UploaderRemark
+        :upload-path="`/datasource/uploadDsSchema/${info.id}`"
+        @upload-finished="init"
+      ></UploaderRemark>
     </div>
     <div class="content">
       <div class="side-list">
@@ -590,8 +656,9 @@ const btnSelectClick = (val: any) => {
     line-height: 22px;
     color: #646a73;
     border-bottom: 1px solid #1f232926;
+    position: relative;
 
-    .ed-button {
+    .ed-button.is-text {
       height: 22px;
       line-height: 22px;
       color: #646a73;
@@ -606,6 +673,12 @@ const btnSelectClick = (val: any) => {
       }
     }
 
+    .export-remark {
+      position: absolute;
+      right: 116px;
+      top: 12px;
+    }
+
     .name {
       color: #1f2329;
       margin-left: 4px;

frontend/src/views/system/excel-upload/UploaderRemark.vue

@@ -0,0 +1,323 @@
+<script setup lang="ts">
+import { computed, ref } from 'vue'
+import { endsWith, startsWith } from 'lodash-es'
+import { useI18n } from 'vue-i18n'
+import icon_warning_filled from '@/assets/svg/icon_warning_filled.svg'
+import icon_upload_outlined from '@/assets/svg/icon_upload_outlined.svg'
+import icon_fileExcel_colorful from '@/assets/datasource/icon_excel.png'
+import { genFileId, type UploadInstance, type UploadProps, type UploadRawFile } from 'element-plus'
+import { useCache } from '@/utils/useCache.ts'
+import { datasourceApi } from '@/api/datasource'
+import { getLocale } from '@/utils/utils.ts'
+import ccmUpload from '@/assets/svg/icon_export_outlined.svg'
+
+const { t } = useI18n()
+const { wsCache } = useCache()
+
+const emits = defineEmits(['upload-finished'])
+
+const props = defineProps<{
+  uploadPath: string
+}>()
+
+const uploadRef = ref<UploadInstance>()
+const uploadLoading = ref(false)
+
+const token = wsCache.get('user.token')
+const locale = getLocale()
+const headers = ref<any>({ 'X-SQLBOT-TOKEN': `Bearer ${token}`, 'Accept-Language': locale })
+const getUploadURL = () => {
+  return import.meta.env.VITE_API_BASE_URL + props.uploadPath
+}
+
+function downloadTemplate() {
+  datasourceApi
+    .exportDsSchema(0)
+    .then((res) => {
+      const blob = new Blob([res], {
+        type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+      })
+      const link = document.createElement('a')
+      link.href = URL.createObjectURL(blob)
+      link.download = '备注.xlsx'
+      document.body.appendChild(link)
+      link.click()
+      document.body.removeChild(link)
+    })
+    .catch(async (error) => {
+      if (error.response) {
+        try {
+          let text = await error.response.data.text()
+          try {
+            text = JSON.parse(text)
+          } finally {
+            ElMessage({
+              message: text,
+              type: 'error',
+              showClose: true,
+            })
+          }
+        } catch (e) {
+          console.error('Error processing error response:', e)
+        }
+      } else {
+        console.error('Other error:', error)
+        ElMessage({
+          message: error,
+          type: 'error',
+          showClose: true,
+        })
+      }
+    })
+}
+
+const handleExceed: UploadProps['onExceed'] = (files) => {
+  uploadRef.value!.clearFiles()
+  const file = files[0] as UploadRawFile
+  file.uid = genFileId()
+  uploadRef.value!.handleStart(file)
+}
+const fileList = ref<Array<any>>([])
+const fileName = computed(() => {
+  if (fileList.value.length > 0) {
+    return fileList.value[0].name
+  }
+  return undefined
+})
+
+const beforeUpload = (rawFile: any) => {
+  if (rawFile.size / 1024 / 1024 > 50) {
+    ElMessage.error(t('common.not_exceed_50mb'))
+    return false
+  }
+  uploadLoading.value = true
+  return true
+}
+
+const errorFileName = ref<string>('')
+
+const infoMessage = ref<string>('')
+const hasError = ref(false)
+
+const onSuccess = (response: any) => {
+  uploadRef.value!.clearFiles()
+  fileList.value = []
+  hasError.value = false
+  infoMessage.value = ''
+  emits('upload-finished', true)
+  if (response?.data?.failed_count > 0 && response?.data?.error_excel_filename) {
+    infoMessage.value = t('training.upload_failed', {
+      success: response.data.success_count,
+      fail: response.data.failed_count,
+      fail_info: '',
+    })
+    errorFileName.value = response.data.error_excel_filename
+    hasError.value = true
+  } else {
+    infoMessage.value = t('training.upload_success')
+    hasError.value = false
+  }
+  uploadLoading.value = false
+  close()
+  showResult()
+}
+
+const onError = (err: Error) => {
+  uploadLoading.value = false
+  uploadRef.value!.clearFiles()
+  fileList.value = []
+  let msg = err.message
+  if (startsWith(msg, '"') && endsWith(msg, '"')) {
+    msg = msg.slice(1, msg.length - 1)
+  }
+  errorFileName.value = ''
+  infoMessage.value = msg
+  hasError.value = true
+  close()
+  showResult()
+}
+
+const dialogShow = ref<boolean>(false)
+
+const resultShow = ref(false)
+
+function showResult() {
+  resultShow.value = true
+}
+
+function open() {
+  dialogShow.value = true
+  errorFileName.value = ''
+  fileList.value = []
+  hasError.value = false
+  infoMessage.value = ''
+}
+
+function close() {
+  uploadRef.value!.clearFiles()
+  fileList.value = []
+  dialogShow.value = false
+}
+
+const submitUpload = () => {
+  uploadRef.value!.submit()
+}
+</script>
+
+<template>
+  <el-button class="up-loader-remark" secondary @click="open">
+    <template #icon>
+      <ccmUpload></ccmUpload>
+    </template>
+    {{ $t('user.import') }}
+  </el-button>
+  <el-dialog
+    v-if="dialogShow"
+    v-model="dialogShow"
+    :title="t('user.batch_import')"
+    width="600px"
+    modal-class="up-loader_dialog"
+    @close="close"
+  >
+    <div class="import-container">
+      <div class="down-template">
+        <span class="icon-span">
+          <el-icon>
+            <Icon name="icon_warning_filled"><icon_warning_filled class="svg-icon" /></Icon>
+          </el-icon>
+        </span>
+        <div class="down-template-content">
+          <span>{{ t('common.upload_hint_first') }}</span>
+          <el-button type="primary" link class="down-button" @click="downloadTemplate">
+            {{ t('common.upload_hint_download_template') }}
+          </el-button>
+          <span>{{ t('common.upload_hint_end') }}</span>
+        </div>
+        <div style="width: 100%; line-height: 22px; padding: 0 26px; margin: -10px 0 8px 0">
+          {{ $t('parameter.memo') }}
+        </div>
+      </div>
+      <div style="width: 100%">
+        <el-upload
+          ref="uploadRef"
+          v-model:file-list="fileList"
+          style="width: 100%"
+          :multiple="false"
+          accept=".xlsx,.xls"
+          :action="getUploadURL()"
+          :before-upload="beforeUpload"
+          :headers="headers"
+          :on-success="onSuccess"
+          :on-error="onError"
+          :show-file-list="false"
+          :auto-upload="false"
+          :limit="1"
+          :on-exceed="handleExceed"
+        >
+          <el-input
+            v-model="fileName"
+            style="width: 100%"
+            :placeholder="t('common.click_to_select_file')"
+            readonly
+          >
+            <template #suffix>
+              <el-icon>
+                <Icon name="icon_upload_outlined">
+                  <icon_upload_outlined class="svg-icon" />
+                </Icon>
+              </el-icon>
+            </template>
+            <template #prefix>
+              <img v-if="!!fileName" :src="icon_fileExcel_colorful" width="16px" height="16px" />
+            </template>
+          </el-input>
+        </el-upload>
+
+        <div>
+          <el-link class="font12" type="info" disabled>
+            {{ t('common.excel_file_type_limit') }}
+          </el-link>
+        </div>
+      </div>
+    </div>
+    <template #footer>
+      <span class="dialog-footer">
+        <el-button @click="close">{{ t('common.cancel') }}</el-button>
+        <el-button
+          :type="fileName ? 'primary' : 'info'"
+          :disabled="!fileName"
+          @click="submitUpload"
+        >
+          {{ t('user.import') }}
+        </el-button>
+      </span>
+    </template>
+  </el-dialog>
+</template>
+
+<style lang="less">
+.up-loader_dialog {
+  .down-template-content {
+    display: flex;
+    flex-direction: row;
+    flex-wrap: wrap;
+    align-items: center;
+    line-height: 24px;
+    height: 24px;
+
+    .down-button {
+      padding: 0;
+      height: unset !important;
+      min-width: unset;
+    }
+  }
+}
+</style>
+<style scoped lang="less">
+.up-loader-remark {
+  margin: 0;
+  position: absolute;
+  top: 12px;
+  right: 24px;
+}
+.import-container {
+  display: flex;
+  flex-direction: column;
+  gap: 16px;
+
+  :deep(.ed-upload) {
+    width: 100%;
+  }
+  .down-template {
+    display: flex;
+    width: 100%;
+    min-height: 40px;
+    flex-wrap: wrap;
+    align-items: center;
+    line-height: 40px;
+    background: var(--ed-color-primary-80, #d2f1e9);
+    border-radius: 4px;
+    padding-left: 10px;
+    .icon-span {
+      color: var(--ed-color-primary);
+      font-size: 18px;
+      i {
+        top: 3px;
+      }
+    }
+    .down-template-content {
+      font-size: 14px;
+      display: flex;
+      flex-direction: row;
+      margin-left: 10px;
+      .down-button {
+        height: 40px;
+      }
+    }
+  }
+
+  .font12 {
+    font-size: 12px;
+  }
+}
+</style>