Merge remote-tracking branch 'origin/main'

.env.example

@@ -1,16 +1,7 @@
-
-# Before running, please copy .env.example to .env
-DOMAIN=localhost
-FRONTEND_HOST=http://localhost:5173
-ENVIRONMENT=local
 PROJECT_NAME="SQLBot"
-STACK_NAME=SQLBot
-
 # Backend
 BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173,https://localhost,https://localhost:5173"
 SECRET_KEY=y5txe1mRmS_JpOrUzFzHEu-kIQn3lf7ll0AOv9DQh0s
-FIRST_SUPERUSER=[email protected]
-FIRST_SUPERUSER_PASSWORD=123456  # Change this to your pwd
 
 TOKEN_KEY="X-SQLBOT-TOKEN"
 DEFAULT_PWD="SQLBot@123456"
@@ -22,7 +13,6 @@ LOG_FORMAT="%(asctime)s - %(name)s - %(levelname)s:%(lineno)d - %(message)s"
 SQL_DEBUG=False
 
 CACHE_TYPE="memory"
-CACHE_REDIS_URL="redis://127.0.0.1:6379"
 
 # Postgres
 POSTGRES_SERVER=localhost
@@ -31,12 +21,6 @@ POSTGRES_DB=sqlbot
 POSTGRES_USER=root
 POSTGRES_PASSWORD=132456 # Change this to your pwd
 
-SENTRY_DSN=
-
-# Configure these with your own Docker registry images
-DOCKER_IMAGE_BACKEND=backend
-DOCKER_IMAGE_FRONTEND=frontend
-
 MCP_IMAGE_PATH=/opt/sqlbot/images
 MCP_IMAGE_HOST=http://localhost:3000
 SERVER_IMAGE_HOST=https://sqlbot.fit2cloud.cn/images/

backend/apps/mcp/mcp.py

@@ -106,7 +106,7 @@ async def mcp_question(session: SessionDep, chat: ChatMcp):
         raise HTTPException(status_code=400, detail="Inactive user")
 
     # ask
-    llm_service = LLMService(session, session_user, chat)
+    llm_service = LLMService(session_user, chat)
     llm_service.init_record()
 
     return StreamingResponse(llm_service.run_task(False), media_type="text/event-stream")

backend/apps/system/api/aimodel.py

@@ -1,5 +1,7 @@
 import json
 from typing import List, Union
+
+from fastapi.responses import StreamingResponse
 from apps.ai_model.model_factory import LLMConfig, LLMFactory
 from apps.system.schemas.ai_model_schema import AiModelConfigItem, AiModelCreator, AiModelEditor, AiModelGridItem
 from fastapi import APIRouter, Query
@@ -14,21 +16,31 @@ router = APIRouter(tags=["system/aimodel"], prefix="/system/aimodel")
 
 @router.post("/status")
 async def check_llm(info: AiModelCreator, trans: Trans):
-    try:
-        additional_params = {item.key: item.val for item in info.config_list}
-        config = LLMConfig(
-            model_type="openai",
-            model_name=info.base_model,
-            api_key=info.api_key,
-            api_base_url=info.api_domain,
-            additional_params=additional_params,
-        )
-        llm_instance = LLMFactory.create_llm(config)
-        result = llm_instance.llm.invoke("who are you?")
-        SQLBotLogUtil.info(f"check_llm result: {result}")
-    except Exception as e:
-        SQLBotLogUtil.error(f"Error checking LLM: {e}")
-        raise Exception(trans('i18n_llm.validate_error', msg = str(e)))
+    async def generate():
+        try:
+            additional_params = {item.key: item.val for item in info.config_list}
+            config = LLMConfig(
+                model_type="openai",
+                model_name=info.base_model,
+                api_key=info.api_key,
+                api_base_url=info.api_domain,
+                additional_params=additional_params,
+            )
+            llm_instance = LLMFactory.create_llm(config)
+            
+            res = llm_instance.llm.stream("who are you?")
+            
+            for chunk in res:
+                if chunk and chunk.content:
+                    SQLBotLogUtil.info(chunk)
+                    yield json.dumps({"content": chunk.content}) + "\n"
+        
+        except Exception as e:
+            SQLBotLogUtil.error(f"Error checking LLM: {e}")
+            error_msg = trans('i18n_llm.validate_error', msg=str(e))
+            yield json.dumps({"error": error_msg}) + "\n"
+    
+    return StreamingResponse(generate(), media_type="application/x-ndjson")
 
 @router.get("", response_model=list[AiModelGridItem])
 async def query(

backend/common/core/config.py

@@ -1,15 +1,11 @@
 import secrets
-import warnings
 from typing import Annotated, Any, Literal
 
 from pydantic import (
     AnyUrl,
     BeforeValidator,
-    EmailStr,
-    HttpUrl,
     PostgresDsn,
     computed_field,
-    model_validator,
 )
 from pydantic_core import MultiHostUrl
 from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -31,12 +27,12 @@ class Settings(BaseSettings):
         env_ignore_empty=True,
         extra="ignore",
     )
+    PROJECT_NAME: str = "SQLBot"
     API_V1_STR: str = "/api/v1"
     SECRET_KEY: str = secrets.token_urlsafe(32)
     # 60 minutes * 24 hours * 8 days = 8 days
     ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8
     FRONTEND_HOST: str = "http://localhost:5173"
-    ENVIRONMENT: Literal["local", "staging", "production"] = "local"
 
     BACKEND_CORS_ORIGINS: Annotated[
         list[AnyUrl] | str, BeforeValidator(parse_cors)
@@ -49,8 +45,6 @@ class Settings(BaseSettings):
             self.FRONTEND_HOST
         ]
 
-    PROJECT_NAME: str
-    SENTRY_DSN: HttpUrl | None = None
     POSTGRES_SERVER: str
     POSTGRES_PORT: int = 5432
     POSTGRES_USER: str
@@ -81,55 +75,8 @@ class Settings(BaseSettings):
             path=self.POSTGRES_DB,
         )
 
-    SMTP_TLS: bool = True
-    SMTP_SSL: bool = False
-    SMTP_PORT: int = 587
-    SMTP_HOST: str | None = None
-    SMTP_USER: str | None = None
-    SMTP_PASSWORD: str | None = None
-    EMAILS_FROM_EMAIL: EmailStr | None = None
-    EMAILS_FROM_NAME: EmailStr | None = None
     MCP_IMAGE_PATH: str
     MCP_IMAGE_HOST: str
     SERVER_IMAGE_HOST: str
 
-    @model_validator(mode="after")
-    def _set_default_emails_from(self) -> Self:
-        if not self.EMAILS_FROM_NAME:
-            self.EMAILS_FROM_NAME = self.PROJECT_NAME
-        return self
-
-    EMAIL_RESET_TOKEN_EXPIRE_HOURS: int = 48
-
-    @computed_field  # type: ignore[prop-decorator]
-    @property
-    def emails_enabled(self) -> bool:
-        return bool(self.SMTP_HOST and self.EMAILS_FROM_EMAIL)
-
-    EMAIL_TEST_USER: EmailStr = "[email protected]"
-    FIRST_SUPERUSER: EmailStr
-    FIRST_SUPERUSER_PASSWORD: str
-
-    def _check_default_secret(self, var_name: str, value: str | None) -> None:
-        if value == "changethis":
-            message = (
-                f'The value of {var_name} is "changethis", '
-                "for security, please change it, at least for deployments."
-            )
-            if self.ENVIRONMENT == "local":
-                warnings.warn(message, stacklevel=1)
-            else:
-                raise ValueError(message)
-
-    @model_validator(mode="after")
-    def _enforce_non_default_secrets(self) -> Self:
-        self._check_default_secret("SECRET_KEY", self.SECRET_KEY)
-        self._check_default_secret("POSTGRES_PASSWORD", self.POSTGRES_PASSWORD)
-        self._check_default_secret(
-            "FIRST_SUPERUSER_PASSWORD", self.FIRST_SUPERUSER_PASSWORD
-        )
-
-        return self
-
-
 settings = Settings()  # type: ignore

backend/main.py

@@ -1,5 +1,4 @@
 from fastapi.concurrency import asynccontextmanager
-import sentry_sdk
 from fastapi import FastAPI
 from fastapi.routing import APIRoute
 from starlette.middleware.cors import CORSMiddleware
@@ -38,9 +37,6 @@ def custom_generate_unique_id(route: APIRoute) -> str:
     return f"{tag}-{route.name}"
 
 
-if settings.SENTRY_DSN and settings.ENVIRONMENT != "local":
-    sentry_sdk.init(dsn=str(settings.SENTRY_DSN), enable_tracing=True)
-
 app = FastAPI(
     title=settings.PROJECT_NAME,
     openapi_url=f"{settings.API_V1_STR}/openapi.json",

backend/template.yaml

@@ -4,9 +4,19 @@ template:
       ### 请使用语言：{lang} 回答，若有深度思考过程，则思考过程也需要使用 {lang} 输出
       
       任务:
-      根据表结构和问题生成符合{engine}数据库引擎规范的sql语句，以及sql中所用到的表名（不要包含schema和database，用数组返回）。
+      根据给定的表结构(M-Schema)和用户问题生成符合{engine}数据库引擎规范的sql语句，以及sql中所用到的表名（不要包含schema和database，用数组返回）。
       你必须遵守以下规则:
+      - 不要编造没有提供给你的表结构
       - 生成的SQL必须符合{engine}的规范。
+      - 你的回答必须使用如下JSON格式返回：
+      {{"success":true,"sql":"生成的SQL语句","tables":["表名1","表名2",...]}}
+      - 问题与生成SQL无关时，直接回答：
+      {{"success":false,"message":"抱歉，我无法回答您的问题。"}}
+      - 如果根据提供的表结构不能生成符合问题与条件的SQL，回答：
+      {{"success":false,"message":"无法生成SQL的原因"}}
+      - 如果问题是图表展示相关且与生成SQL查询无关时，请参考上一次回答的SQL来生成SQL
+      - 如果问题是图表展示相关，可参考的图表类型为表格(table)、柱状图(column)、条形图(bar)、折线图(line)或饼图(pie)，返回的JSON：
+       {{"success":true,"sql":"生成的SQL语句","chart-type":"选择的图表类型(table/column/bar/line/pie)","tables":["表名1","表名2",...]}}
       - 提问中如果有涉及数据源名称或数据源描述的内容，则忽略数据源的信息，直接根据剩余内容生成SQL
       - 根据表结构生成SQL语句，需给每个表名生成一个别名（不要加AS）。
       - SQL查询中不能使用星号(*)，必须明确指定字段名.
@@ -18,9 +28,9 @@ template:
       - 如数据库引擎是 MySQL，则在表名、字段名、别名外层加反引号；
       - 如数据库引擎是 Microsoft SQL Server，则在schema、表名、字段名、别名外层加方括号。
       - 以PostgreSQL为例，查询Schema为TEST表TABLE下所有数据，则生成的SQL为：
-          SELECT * FROM "TEST"."TABLE"
+          SELECT "id" FROM "TEST"."TABLE"
           - 注意在表名外双引号的位置，千万不要生成为:
-            SELECT * FROM "TEST.TABLE"
+            SELECT "id" FROM "TEST.TABLE"
       - 如果生成SQL的字段内有时间格式的字段（重要）:
           - 若提问中没有指定查询顺序，则默认按时间升序排序
           - 若提问是时间，且没有指定具体格式，则格式化为yyyy-MM-dd HH:mm:ss的格式
@@ -28,23 +38,27 @@ template:
           - 若提问是年月，且没有指定具体格式，则格式化为yyyy-MM的格式
           - 若提问是年，且没有指定具体格式，则格式化为yyyy的格式
           - 生成的格式化语法需要适配对应的数据库引擎。
-      - 生成的SQL使用JSON格式返回：
-      {{"success":true,"sql":"生成的SQL语句","tables":["表名1","表名2",...]}}
-      - 问题与生成SQL无关时，直接回答：
-      {{"success":false,"message":"抱歉，我无法回答您的问题。"}}
-      - 如果根据提供的表结构不能生成符合问题与条件的SQL，回答：
-      {{"success":false,"message":"无法生成SQL的原因"}}
-      - 如果问题是图表展示相关且与生成SQL查询无关时，请参考上一次回答的SQL来生成SQL
-      - 如果问题是图表展示相关，可参考的图表类型为表格(table)、柱状图(column)、条形图(bar)、折线图(line)或饼图(pie)，返回的JSON：
-       {{"success":true,"sql":"生成的SQL语句","chart-type":"选择的图表类型(table/column/bar/line/pie)","tables":["表名1","表名2",...]}}
       - 生成的SQL查询结果可以用来进行图表展示，需要注意排序字段的排序优先级，例如：
           - 柱状图或折线图：适合展示在横轴的字段优先排序，若SQL包含分类字段，则分类字段次一级排序
       
+      ### M-Schema格式简单的解释如下:
+      ```
+      【DB_ID】 [Database名]
+      【Schema】
+      # Table: [Database名].[Table名], [表描述（若没有则为空）]
+      [
+      ([字段名1]:[字段1的类型], [字段1的描述（这一行的逗号后都是描述，若没有则为空）]),
+      ([字段名2]:[字段2的类型], [字段2的描述（这一行的逗号后都是描述，若没有则为空）]),
+      ([字段名3]:[字段3的类型], [字段3的描述（这一行的逗号后都是描述，若没有则为空）]),
+      ...
+      ]
+      ```
+      
       ### 响应, 请直接返回JSON结果:
       ```json
 
     user: |
-      ### 表结构:
+      ### 提供表结构如下:
       {schema}
       
       ### 问题:

frontend/src/api/system.ts

@@ -8,6 +8,5 @@ export const modelApi = {
   delete: (id: number) => request.delete(`/system/aimodel/${id}`),
   query: (id: number) => request.get(`/system/aimodel/${id}`),
   setDefault: (id: number) => request.put(`/system/aimodel/default/${id}`),
-  check: (data: any) =>
-    request.post('/system/aimodel/status', data, { requestOptions: { silent: true } }),
+  check: (data: any) => request.fetchStream('/system/aimodel/status', data),
 }

frontend/src/views/system/model/Card.vue

@@ -2,7 +2,7 @@
 import delIcon from '@/assets/svg/icon_delete.svg'
 import edit from '@/assets/svg/icon_edit_outlined.svg'
 import { get_supplier } from '@/entity/supplier'
-import { computed } from 'vue'
+import { computed, ref } from 'vue'
 const props = withDefaults(
   defineProps<{
     name: string
@@ -21,12 +21,22 @@ const props = withDefaults(
     supplier: 0,
   }
 )
+const errorMsg = ref('')
 const current_supplier = computed(() => {
   if (!props.supplier) {
     return null
   }
   return get_supplier(props.supplier)
 })
+const showErrorMask = (msg?: string) => {
+  if (!msg) {
+    return
+  }
+  errorMsg.value = msg
+  setTimeout(() => {
+    errorMsg.value = ''
+  }, 3000)
+}
 const emits = defineEmits(['edit', 'del'])
 
 const handleEdit = () => {
@@ -36,10 +46,17 @@ const handleEdit = () => {
 const handleDel = () => {
   emits('del', { id: props.id, name: props.name, default_model: props.isDefault })
 }
+
+defineExpose({ showErrorMask })
 </script>
 
 <template>
-  <div class="card">
+  <div
+    v-loading="!!errorMsg"
+    class="card"
+    :element-loading-text="errorMsg"
+    element-loading-custom-class="model-card-loading"
+  >
     <div class="name-icon">
       <img :src="current_supplier?.icon" width="32px" height="32px" />
       <span :title="name" class="name ellipsis">{{ name }}</span>
@@ -163,5 +180,29 @@ const handleDel = () => {
       display: flex;
     }
   }
+  :deep(.model-card-loading) {
+    border-radius: 12px;
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-end;
+    align-items: end;
+    background-color: rgb(122 122 122 / 87%);
+    .ed-loading-spinner {
+      top: auto;
+      margin: 8px 4px;
+      display: flex;
+      position: relative;
+      justify-content: flex-end;
+      align-items: center;
+      width: calc(100% - 8px);
+    }
+    svg {
+      display: none;
+    }
+    p {
+      text-align: left;
+      color: var(--ed-color-danger);
+    }
+  }
 }
 </style>

frontend/src/views/system/model/Model.vue

@@ -34,7 +34,8 @@ const activeStep = ref(0)
 const activeName = ref('')
 const activeType = ref('')
 const modelFormRef = ref()
-
+const cardRefs = ref<any[]>([])
+const showCardError = ref(true) // if you don`t want card mask error, just change this to false
 reactive({
   form: {
     id: '',
@@ -69,16 +70,46 @@ const defaultModelListWithSearch = computed(() => {
   })
 })
 
-const modelCheckHandler = (item: any) => {
+const modelCheckHandler = async (item: any) => {
+  const response = await modelApi.check(item)
+  const reader = response.body.getReader()
+  const decoder = new TextDecoder()
+  let checkTimeout = false
   setTimeout(() => {
-    modelApi.check(item).catch((err: any) => {
-      if (err.response?.data?.msg) {
-        ElMessage.error(t('model.check_failed', { msg: err.response.data.msg || '' }))
-      } else {
-        ElMessage.error(t('model.check_failed', { msg: err.message || '' }))
+    checkTimeout = true
+  }, 9000)
+  let checkMsg = ''
+  while (true) {
+    if (checkTimeout) {
+      break
+    }
+    const { done, value } = await reader.read()
+    if (done) break
+    const lines = decoder.decode(value).trim().split('\n')
+    for (const line of lines) {
+      const data = JSON.parse(line)
+      if (data.error) {
+        checkMsg += data.error
+      } else if (data.content) {
+        console.debug(data.content)
       }
-    })
-  }, 1000)
+    }
+  }
+  if (!checkMsg) {
+    return
+  }
+  console.error(checkMsg)
+  if (!showCardError.value) {
+    ElMessage.error(checkMsg)
+    return
+  }
+  nextTick(() => {
+    const index = modelListWithSearch.value.findIndex((el: any) => el.id === item.id)
+    if (index > -1) {
+      const currentRef = cardRefs.value[index]
+      currentRef?.showErrorMask(checkMsg)
+    }
+  })
 }
 const duplicateName = async (item: any) => {
   const res = await modelApi.queryAll()
@@ -218,7 +249,11 @@ const preStep = () => {
 const saveModel = () => {
   modelFormRef.value.submitModel()
 }
-
+const setCardRef = (el: any, index: number) => {
+  if (el) {
+    cardRefs.value[index] = el
+  }
+}
 const search = () => {
   modelApi.queryAll().then((res: any) => {
     modelList.value = res
@@ -308,8 +343,9 @@ const submit = (item: any) => {
 
     <div v-else class="card-content">
       <card
-        v-for="ele in modelListWithSearch"
+        v-for="(ele, index) in modelListWithSearch"
         :id="ele.id"
+        :ref="(el: any) => setCardRef(el, index)"
         :key="ele.id"
         :name="ele.name"
         :supplier="ele.supplier"