Merge branch 'main' of https://github.com/dataease/SQLBot

backend/alembic/env.py

@@ -24,7 +24,8 @@ config = context.config
 
 # from apps.system.models.user import SQLModel  # noqa
 # from apps.settings.models.setting_models import SQLModel
-from apps.chat.models.chat_model import SQLModel
+# from apps.chat.models.chat_model import SQLModel
+from apps.terminology.models.terminology_model import SQLModel
 # from apps.dashboard.models.dashboard_model import SQLModel
 from common.core.config import settings # noqa
 

backend/alembic/versions/039_create_terminology.py

@@ -0,0 +1,41 @@
+"""039_create_terminology
+
+Revision ID: 25cbc85766fd
+Revises: fc23c4f3e755
+Create Date: 2025-08-25 11:38:32.990973
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
+import pgvector
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '25cbc85766fd'
+down_revision = 'fc23c4f3e755'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute("CREATE EXTENSION IF NOT EXISTS vector;")
+
+    op.create_table('terminology',
+    sa.Column('id', sa.BigInteger(), sa.Identity(always=True), nullable=False),
+    sa.Column('pid', sa.BigInteger(), nullable=True),
+    sa.Column('create_time', sa.DateTime(), nullable=True),
+    sa.Column('word', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True),
+    sa.Column('description', sa.Text(), nullable=True),
+    sa.Column('embedding', pgvector.sqlalchemy.vector.VECTOR(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('terminology')
+    # ### end Alembic commands ###

backend/apps/ai_model/embedding.py

@@ -0,0 +1,59 @@
+import threading
+from typing import Optional
+
+from langchain_core.embeddings import Embeddings
+from langchain_huggingface import HuggingFaceEmbeddings
+from pydantic import BaseModel
+
+from common.core.config import settings
+
+
+class EmbeddingModelInfo(BaseModel):
+    folder: str
+    name: str
+    device: str = 'cpu'
+
+
+local_embedding_model = EmbeddingModelInfo(folder=settings.LOCAL_MODEL_PATH, name=settings.DEFAULT_EMBEDDING_MODEL)
+
+_lock = threading.Lock()
+locks = {}
+
+_embedding_model: dict[str, Optional[Embeddings]] = {}
+
+
+class EmbeddingModelCache:
+
+    @staticmethod
+    def _new_instance(config: EmbeddingModelInfo = local_embedding_model):
+        return HuggingFaceEmbeddings(model_name=config.name, cache_folder=config.folder,
+                                     model_kwargs={'device': config.device},
+                                     encode_kwargs={'normalize_embeddings': True}
+                                     )
+
+    @staticmethod
+    def _get_lock(key: str = settings.DEFAULT_EMBEDDING_MODEL):
+        lock = locks.get(key)
+        if lock is None:
+            with _lock:
+                lock = locks.get(key)
+                if lock is None:
+                    lock = threading.Lock()
+                    locks[key] = lock
+
+        return lock
+
+    @staticmethod
+    def get_model(key: str = settings.DEFAULT_EMBEDDING_MODEL,
+                  config: EmbeddingModelInfo = local_embedding_model) -> Embeddings:
+        global _embedding_model
+        model_instance = _embedding_model.get(key)
+        if model_instance is None:
+            lock = EmbeddingModelCache._get_lock(key)
+            with lock:
+                model_instance = _embedding_model.get(key)
+                if model_instance is None:
+                    model_instance = EmbeddingModelCache._new_instance(config)
+                    _embedding_model[key] = model_instance
+
+        return model_instance

backend/apps/api.py

@@ -1,9 +1,9 @@
 from fastapi import APIRouter
 
+from apps.terminology.api import terminology
 from apps.chat.api import chat
 from apps.dashboard.api import dashboard_api
 from apps.datasource.api import datasource
-from apps.settings.api import terminology
 from apps.system.api import login, user, aimodel, workspace, assistant
 from apps.mcp import mcp
 
@@ -19,3 +19,4 @@ api_router.include_router(chat.router)
 api_router.include_router(dashboard_api.router)
 api_router.include_router(mcp.router)
 
+

backend/apps/terminology/api/terminology.py

@@ -0,0 +1,37 @@
+from typing import Optional
+
+from fastapi import APIRouter, Query
+
+from apps.terminology.curd.terminology import page_terminology, create_terminology, update_terminology, \
+    delete_terminology
+from apps.terminology.models.terminology_model import TerminologyInfo
+from common.core.deps import SessionDep
+
+router = APIRouter(tags=["Terminology"], prefix="/system/terminology")
+
+
+@router.get("/page/{current_page}/{page_size}")
+async def pager(session: SessionDep, current_page: int, page_size: int,
+                word: Optional[str] = Query(None, description="搜索术语(可选)")):
+    current_page, page_size, total_count, total_pages, _list = page_terminology(session, current_page, page_size, word)
+
+    return {
+        "current_page": current_page,
+        "page_size": page_size,
+        "total_count": total_count,
+        "total_pages": total_pages,
+        "data": _list
+    }
+
+
+@router.put("")
+async def create_or_update(session: SessionDep, info: TerminologyInfo):
+    if info.id:
+        return update_terminology(session, info)
+    else:
+        return create_terminology(session, info)
+
+
+@router.delete("")
+async def delete(session: SessionDep, id_list: list[int]):
+    delete_terminology(session, id_list)

backend/apps/terminology/curd/terminology.py

@@ -0,0 +1,196 @@
+import datetime
+from typing import List, Optional
+
+from sqlalchemy import and_, or_, select, func, delete, update
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import aliased
+
+from apps.terminology.models.terminology_model import Terminology, TerminologyInfo
+from common.core.deps import SessionDep
+
+
+def page_terminology(session: SessionDep, current_page: int = 1, page_size: int = 10, name: Optional[str] = None):
+    _list: List[TerminologyInfo] = []
+
+    child = aliased(Terminology)
+
+    current_page = max(1, current_page)
+    page_size = max(10, page_size)
+
+    total_count = 0
+    total_pages = 0
+
+    if name and name.strip() != "":
+        keyword_pattern = f"%{name.strip()}%"
+        # 步骤1：先找到所有匹配的节点ID（无论是父节点还是子节点）
+        matched_ids_subquery = (
+            select(Terminology.id)
+            .where(Terminology.name.like(keyword_pattern))  # LIKE查询条件
+            .subquery()
+        )
+
+        # 步骤2：找到这些匹配节点的所有父节点（包括自身如果是父节点）
+        parent_ids_subquery = (
+            select(Terminology.id)
+            .where(
+                (Terminology.id.in_(matched_ids_subquery)) |
+                (Terminology.id.in_(
+                    select(Terminology.pid)
+                    .where(Terminology.id.in_(matched_ids_subquery))
+                    .where(Terminology.pid.isnot(None))
+                ))
+            )
+            .where(Terminology.pid.is_(None))  # 只取父节点
+        )
+
+        count_stmt = select(func.count()).select_from(parent_ids_subquery.subquery())
+        total_count = session.execute(count_stmt).scalar()
+        total_pages = (total_count + page_size - 1) // page_size
+
+        # 步骤3：获取分页后的父节点ID
+        paginated_parent_ids = (
+            parent_ids_subquery
+            .order_by(Terminology.create_time.desc())
+            .offset((current_page - 1) * page_size)
+            .limit(page_size)
+            .subquery()
+        )
+
+        # 步骤4：获取这些父节点的childrenNames
+        children_subquery = (
+            select(
+                child.pid,
+                func.jsonb_agg(child.name).order_by(child.create_time.desc()).label('children_names')
+            )
+            .where(child.pid.isnot(None))
+            .group_by(child.pid)
+            .subquery()
+        )
+
+        # 主查询
+        stmt = (
+            select(
+                Terminology.id,
+                Terminology.word,
+                Terminology.create_time,
+                Terminology.description,
+                func.coalesce(
+                    children_subquery.c.children_names,
+                    func.cast('[]', JSONB)
+                ).label('other_words')
+            )
+            .outerjoin(
+                children_subquery,
+                Terminology.id == children_subquery.c.pid
+            )
+            .where(Terminology.id.in_(paginated_parent_ids))
+            .order_by(Terminology.create_time.desc())
+        )
+        print(str(stmt))
+    else:
+        parent_ids_subquery = (
+            select(Terminology.id)
+            .where(Terminology.pid.is_(None))  # 只取父节点
+        )
+        count_stmt = select(func.count()).select_from(parent_ids_subquery.subquery())
+        total_count = session.execute(count_stmt).scalar()
+        total_pages = (total_count + page_size - 1) // page_size
+
+        paginated_parent_ids = (
+            parent_ids_subquery
+            .order_by(Terminology.create_time.desc())
+            .offset((current_page - 1) * page_size)
+            .limit(page_size)
+            .subquery()
+        )
+
+        stmt = (
+            select(
+                Terminology.id,
+                Terminology.word,
+                Terminology.create_time,
+                Terminology.description,
+                func.coalesce(
+                    func.jsonb_agg(child.word),
+                    func.cast('[]', JSONB)
+                ).label('other_words')
+            )
+            .outerjoin(child, and_(Terminology.id == child.pid))
+            .where(Terminology.id.in_(paginated_parent_ids))
+            .group_by(Terminology.id, Terminology.word)
+            .order_by(Terminology.create_time.desc())
+        )
+        print(str(stmt))
+
+    result = session.execute(stmt)
+
+    for row in result:
+        _list.append(TerminologyInfo(
+            id=row.id,
+            word=row.word,
+            create_time=row.create_time,
+            description=row.description,
+            other_words=row.other_words,
+        ))
+
+    return current_page, page_size, total_count, total_pages, _list
+
+
+def create_terminology(session: SessionDep, info: TerminologyInfo):
+    create_time = datetime.datetime.now()
+    parent = Terminology(word=info.word, create_time=create_time, description=info.description)
+
+    result = Terminology(**parent.model_dump())
+
+    session.add(parent)
+    session.flush()
+    session.refresh(parent)
+
+    result.id = parent.id
+    session.commit()
+
+    _list: List[Terminology] = []
+    if info.other_words:
+        for other_word in info.other_words:
+            _list.append(
+                Terminology(pid=result.id, word=other_word, create_time=create_time, description=info.description))
+    session.bulk_save_objects(_list)
+    session.flush()
+    session.commit()
+
+    # todo embedding
+
+    return result.id
+
+
+def update_terminology(session: SessionDep, info: TerminologyInfo):
+    stmt = update(Terminology).where(and_(Terminology.id == info.id)).values(
+        word=info.word,
+        description=info.description,
+    )
+    session.execute(stmt)
+    session.commit()
+
+    stmt = delete(Terminology).where(and_(Terminology.pid == info.id))
+    session.execute(stmt)
+    session.commit()
+
+    create_time = datetime.datetime.now()
+    _list: List[Terminology] = []
+    if info.other_words:
+        for other_word in info.other_words:
+            _list.append(
+                Terminology(pid=info.id, word=other_word, create_time=create_time, description=info.description))
+    session.bulk_save_objects(_list)
+    session.flush()
+    session.commit()
+
+    # todo embedding
+
+    return info.id
+
+
+def delete_terminology(session: SessionDep, ids: list[int]):
+    stmt = delete(Terminology).where(or_(Terminology.id.in_(ids), Terminology.pid.in_(ids)))
+    session.execute(stmt)
+    session.commit()

backend/apps/terminology/models/terminology_model.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+from typing import List, Optional
+
+from pgvector.sqlalchemy import VECTOR
+from pydantic import BaseModel
+from sqlalchemy import Column, Text, BigInteger, DateTime, Identity
+from sqlmodel import SQLModel, Field
+
+
+class Terminology(SQLModel, table=True):
+    __tablename__ = "terminology"
+    id: Optional[int] = Field(sa_column=Column(BigInteger, Identity(always=True), primary_key=True))
+    pid: Optional[int] = Field(sa_column=Column(BigInteger, nullable=True))
+    create_time: Optional[datetime] = Field(sa_column=Column(DateTime(timezone=False), nullable=True))
+    word: Optional[str] = Field(max_length=255)
+    description: Optional[str] = Field(sa_column=Column(Text, nullable=True))
+    embedding: Optional[List[float]] = Field(sa_column=Column(VECTOR(), nullable=True))
+
+
+class TerminologyInfo(BaseModel):
+    id: Optional[int] = None
+    create_time: Optional[datetime] = None
+    word: Optional[str] = None
+    description: Optional[str] = None
+    other_words: Optional[List[str]] = []
+
+

backend/common/core/config.py

@@ -50,20 +50,20 @@ class Settings(BaseSettings):
     POSTGRES_PASSWORD: str = "123456"
     POSTGRES_DB: str = "sqlbot"
     SQLBOT_DB_URL: str = ''
-    #SQLBOT_DB_URL: str = 'mysql+pymysql://root:Password123%[email protected]:3306/sqlbot'
-    
-    TOKEN_KEY: str =  "X-SQLBOT-TOKEN"
+    # SQLBOT_DB_URL: str = 'mysql+pymysql://root:Password123%[email protected]:3306/sqlbot'
+
+    TOKEN_KEY: str = "X-SQLBOT-TOKEN"
     DEFAULT_PWD: str = "SQLBot@123456"
     ASSISTANT_TOKEN_KEY: str = "X-SQLBOT-ASSISTANT-TOKEN"
-    
+
     CACHE_TYPE: Literal["redis", "memory", "None"] = "memory"
-    CACHE_REDIS_URL: str | None = None # Redis URL, e.g., "redis://[[username]:[password]]@localhost:6379/0"
-    
+    CACHE_REDIS_URL: str | None = None  # Redis URL, e.g., "redis://[[username]:[password]]@localhost:6379/0"
+
     LOG_LEVEL: str = "INFO"  # DEBUG, INFO, WARNING, ERROR
     LOG_DIR: str = "logs"
     LOG_FORMAT: str = "%(asctime)s - %(name)s - %(levelname)s:%(lineno)d - %(message)s"
     SQL_DEBUG: bool = False
-    
+
     UPLOAD_DIR: str = "/opt/sqlbot/data/file"
     SQLBOT_KEY_EXPIRED: int = 100  # License key expiration timestamp, 0 means no expiration
 
@@ -86,4 +86,11 @@ class Settings(BaseSettings):
     MCP_IMAGE_HOST: str = 'http://localhost:3000'
     SERVER_IMAGE_HOST: str = ''
 
+    LOCAL_MODEL_PATH: str = '/opt/sqlbot/models'
+    DEFAULT_EMBEDDING_MODEL: str = 'shibing624/text2vec-base-chinese'
+
+    EMBEDDING_SIMILARITY: float = 0.6
+    EMBEDDING_TOP_COUNT: int = 3
+
+
 settings = Settings()  # type: ignore

backend/pyproject.toml

@@ -23,7 +23,9 @@ dependencies = [
     "langchain-core>=0.3,<0.4",
     "langchain-openai>=0.3,<0.4",
     "langchain-community>=0.3,<0.4",
+    "langchain-huggingface>=0.2.0",
     "langgraph>=0.3,<0.4",
+    "pgvector>=0.4.1",
     "dashscope>=1.14.0,<2.0.0",
     "pymysql (>=1.1.1,<2.0.0)",
     "cryptography (>=44.0.3,<45.0.0)",

frontend/src/i18n/en.json

@@ -644,4 +644,4 @@
     "setting_successfully": "Setting Successfully",
     "customize_theme_color": "Customize theme color"
   }
-}
+}

frontend/src/i18n/zh-CN.json

@@ -644,4 +644,4 @@
     "setting_successfully": "设置成功",
     "customize_theme_color": "自定义主题色"
   }
-}
+}

frontend/src/views/system/professional/index.vue

@@ -321,7 +321,7 @@ const deleteHandlerItem = (idx: number) => {
     <div
       v-if="!searchLoading"
       class="table-content"
-      :class="multipleSelectionAll.length && 'show-pagination_height'"
+      :class="multipleSelectionAll?.length && 'show-pagination_height'"
     >
       <template v-if="!keywords && !fieldList.length">
         <EmptyBackground