From 551d5359f13e8240839a6aa0f26d0bff65be9725 Mon Sep 17 00:00:00 2001 From: dataeaseShu Date: Fri, 23 May 2025 11:10:35 +0800 Subject: [PATCH 1/6] feat(datasource): Datasource table virtualization --- .gitignore | 1 + frontend/src/style.less | 48 +++- frontend/src/views/ds/form.vue | 459 ++++++++++++++++++-------------- frontend/src/views/ds/index.vue | 1 - 4 files changed, 308 insertions(+), 201 deletions(-) diff --git a/.gitignore b/.gitignore index 752ee168..237d53ba 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ node_modules/ /playwright/.cache/ *.lock *-lock.json +components.d.ts # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/frontend/src/style.less b/frontend/src/style.less index 7ac20dd3..a0cdd12e 100644 --- a/frontend/src/style.less +++ b/frontend/src/style.less @@ -12,9 +12,8 @@ -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; - /* --primary-color: var(--el-color-primary); */ - --primary-color: #1CBA90; + --primary-color: #1cba90; --primary-light: #e8f0fe; --secondary-color: #5f6368; --danger-color: #ea4335; @@ -30,7 +29,7 @@ --border-radius: 8px; --shadow: 0 1px 3px rgba(0, 0, 0, 0.1); - --el-color-primary: #1CBA90 !important; + --el-color-primary: #1cba90 !important; } a { @@ -50,6 +49,12 @@ body { min-height: 100vh; } +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + #app { width: 100%; margin: 0; @@ -67,3 +72,40 @@ body { background-color: #f9f9f9; } } + +.ellipsis { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.list-item_primary { + height: 40px; + border-radius: 4px; + padding: 8px 12px; + cursor: pointer; + display: flex; + align-items: center; + + .ed-icon + .label { + margin-left: 5.33px; + } + + .label { + width: 85%; + font-size: 14px; + .ellipsis; + } + + &:not(.active):hover { + background: rgba(31, 35, 41, 0.1); + } + + &.active { + background: var(--ed-color-primary-1a, rgba(51, 112, 255, 0.1)); + color: var(--ed-color-primary); + } +} + + + diff --git a/frontend/src/views/ds/form.vue b/frontend/src/views/ds/form.vue index c0eec9bd..ba6b09df 100644 --- a/frontend/src/views/ds/form.vue +++ b/frontend/src/views/ds/form.vue @@ -2,12 +2,18 @@ - + @@ -20,7 +26,11 @@ - + - Get Schema + Get Schema Supported version: 2012+ Supported version: 12+ -
- - - - - {{ item.tableName }} - - - - +
+ + + + + Selected: {{ checkList.length }}/{{ tableList.length }}
-
+
Cancel - Test Connect - Next - Preview - Save + Test Connect + Next + Preview + Save
- \ No newline at end of file + diff --git a/frontend/src/views/ds/index.vue b/frontend/src/views/ds/index.vue index 06bc3ec8..3e9a57c0 100644 --- a/frontend/src/views/ds/index.vue +++ b/frontend/src/views/ds/index.vue @@ -180,7 +180,6 @@ onMounted(() => { position: relative; border: 1px solid #dadce0; transition: all 0.2s ease; - height: 110px; align-items: center; } .connection-icon { From b50ee7e2c2169a72b3e9b01d03915ad8f743d277 Mon Sep 17 00:00:00 2001 From: junjun Date: Fri, 23 May 2025 11:33:30 +0800 Subject: [PATCH 2/6] feat(datasource): datasource UI --- frontend/src/views/ds/form.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/views/ds/form.vue b/frontend/src/views/ds/form.vue index ba6b09df..18370e14 100644 --- a/frontend/src/views/ds/form.vue +++ b/frontend/src/views/ds/form.vue @@ -91,6 +91,8 @@ Supported version: 2012+ Supported version: 12+ + Supported version: 5.6+ + Supported version: 9.6+
From e279ae4a36cb5928b6ab2a277fe459d33da06825 Mon Sep 17 00:00:00 2001 From: ulleo Date: Fri, 23 May 2025 13:55:23 +0800 Subject: [PATCH 3/6] feat: Q&A Chat --- backend/alembic/env.py | 3 +- backend/apps/chat/api/chat.py | 86 ++- backend/apps/chat/curd/__init__.py | 0 backend/apps/chat/curd/chat.py | 132 +++++ backend/apps/chat/models/__init__.py | 0 backend/apps/chat/models/chat_model.py | 49 ++ backend/apps/chat/schemas/chat_schema.py | 3 +- backend/pyproject.toml | 2 +- frontend/package.json | 3 +- frontend/src/api/chat.ts | 167 +++++- frontend/src/style.less | 3 - frontend/src/utils/utils.ts | 45 +- frontend/src/views/chat/index.vue | 632 ++++++++--------------- 13 files changed, 671 insertions(+), 454 deletions(-) create mode 100644 backend/apps/chat/curd/__init__.py create mode 100644 backend/apps/chat/curd/chat.py create mode 100644 backend/apps/chat/models/__init__.py create mode 100644 backend/apps/chat/models/chat_model.py diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 685e1d8a..ce655b09 100755 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -25,7 +25,8 @@ fileConfig(config.config_file_name) # target_metadata = None # from apps.system.models.user import SQLModel # noqa -from apps.settings.models.setting_models import SQLModel +# from apps.settings.models.setting_models import SQLModel +from apps.chat.models.chat_model import SQLModel from common.core.config import settings # noqa target_metadata = SQLModel.metadata diff --git a/backend/apps/chat/api/chat.py b/backend/apps/chat/api/chat.py index 8d944033..4da08b13 100644 --- a/backend/apps/chat/api/chat.py +++ b/backend/apps/chat/api/chat.py @@ -1,37 +1,68 @@ from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse from sqlmodel import select + +from apps.chat.curd.chat import list_chats, get_chat_with_records, create_chat, save_question, save_answer +from apps.chat.models.chat_model import CreateChat, ChatRecord from apps.chat.schemas.chat_base_schema import LLMConfig from apps.chat.schemas.chat_schema import ChatQuestion from apps.chat.schemas.llm import AgentService from apps.datasource.models.datasource import CoreDatasource from apps.system.models.system_model import AiModelDetail -from common.core.deps import SessionDep +from common.core.deps import SessionDep, CurrentUser import json import asyncio router = APIRouter(tags=["Data Q&A"], prefix="/chat") +@router.get("/list") +async def chats(session: SessionDep, current_user: CurrentUser): + return list_chats(session, current_user) + + +@router.get("/get/{chart_id}") +async def list_chat(session: SessionDep, current_user: CurrentUser, chart_id: int): + try: + return get_chat_with_records(chart_id=chart_id, session=session, current_user=current_user) + except Exception as e: + raise HTTPException( + status_code=500, + detail=str(e) + ) + + +@router.post("/start") +async def start_chat(session: SessionDep, current_user: CurrentUser, create_chat_obj: CreateChat): + try: + return create_chat(session, current_user, create_chat_obj) + except Exception as e: + raise HTTPException( + status_code=400, + detail=str(e) + ) + + @router.post("/question") -async def stream_sql(session: SessionDep, requestQuestion: ChatQuestion): +async def stream_sql(session: SessionDep, current_user: CurrentUser, request_question: ChatQuestion): """Stream SQL analysis results Args: session: Database session - requestQuestion: User question model + current_user: CurrentUser + request_question: User question model Returns: Streaming response with analysis results """ - question = requestQuestion.question - + question = request_question.question + # Get available AI model aimodel = session.exec(select(AiModelDetail).where( - AiModelDetail.status == True, + AiModelDetail.status == True, AiModelDetail.api_key.is_not(None) )).first() - + # Get available datasource ds = session.exec(select(CoreDatasource).where( CoreDatasource.status == 'Success' @@ -42,13 +73,22 @@ async def stream_sql(session: SessionDep, requestQuestion: ChatQuestion): status_code=400, detail="No available AI model configuration found" ) - + if not ds: raise HTTPException( status_code=400, detail="No available datasource configuration found" ) - + + record: ChatRecord + try: + record = save_question(session=session, current_user=current_user, question=request_question) + except Exception as e1: + raise HTTPException( + status_code=400, + detail=str(e1) + ) + # Use Tongyi Qianwen tongyi_config = LLMConfig( model_type="openai", @@ -72,25 +112,37 @@ async def stream_sql(session: SessionDep, requestQuestion: ChatQuestion): vllm_service = LLMService(vllm_config) """ """ result = llm_service.generate_sql(question) return result """ - + async def event_generator(): + all_text = '' try: async for chunk in llm_service.async_generate(question): data = json.loads(chunk.replace('data: ', '')) - + if data['type'] in ['final', 'tool_result']: content = data['content'] + print('-- ' + content) + all_text += content for char in content: yield f"data: {json.dumps({'type': 'char', 'content': char})}\n\n" - await asyncio.sleep(0.05) - + await asyncio.sleep(0.05) + if 'html' in data: yield f"data: {json.dumps({'type': 'html', 'content': data['html']})}\n\n" else: yield chunk - + except Exception as e: + all_text = 'Exception:' + str(e) yield f"data: {json.dumps({'type': 'error', 'content': str(e)})}\n\n" - - #return EventSourceResponse(event_generator(), headers={"Content-Type": "text/event-stream"}) - return StreamingResponse(event_generator(), media_type="text/event-stream") \ No newline at end of file + + try: + save_answer(session=session, id=record.id, answer=all_text) + except Exception as e: + raise HTTPException( + status_code=500, + detail=str(e) + ) + + # return EventSourceResponse(event_generator(), headers={"Content-Type": "text/event-stream"}) + return StreamingResponse(event_generator(), media_type="text/event-stream") diff --git a/backend/apps/chat/curd/__init__.py b/backend/apps/chat/curd/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/chat/curd/chat.py b/backend/apps/chat/curd/chat.py new file mode 100644 index 00000000..a59e185d --- /dev/null +++ b/backend/apps/chat/curd/chat.py @@ -0,0 +1,132 @@ +import datetime +import json +from typing import List + +from sqlalchemy import text, and_ +from sqlmodel import select + +from apps.chat.models.chat_model import Chat, ChatRecord, CreateChat, ChatInfo +from apps.chat.schemas.chat_schema import ChatQuestion +from apps.datasource.models.datasource import CoreDatasource +from common.core.deps import SessionDep, CurrentUser + + +def list_chats(session: SessionDep, current_user: CurrentUser) -> List[Chat]: + chart_list = session.query(Chat).filter(Chat.create_by == current_user.id).order_by( + Chat.create_time.desc()).all() + return chart_list + + +def get_chat_with_records(session: SessionDep, chart_id: int, current_user: CurrentUser) -> ChatInfo: + chat = session.query(Chat).filter(Chat.id == chart_id).first() + if not chat: + raise Exception(f"Chat with id {chart_id} not found") + + chat_info = ChatInfo(**chat.model_dump()) + + ds = session.query(CoreDatasource).filter(CoreDatasource.id == chat.datasource).first() + if not ds: + chat_info.datasource_exists = False + chat_info.datasource_name = 'Datasource not exist' + else: + chat_info.datasource_exists = True + chat_info.datasource_name = ds.name + + record_list = session.query(ChatRecord).filter( + and_(Chat.create_by == current_user.id, ChatRecord.id == chart_id)).order_by(ChatRecord.create_time).all() + + chat_info.records = record_list + + return chat_info + + +def create_chat(session: SessionDep, current_user: CurrentUser, create_chat_obj: CreateChat) -> ChatInfo: + if not create_chat_obj.datasource: + raise Exception("Datasource cannot be None") + + if not create_chat_obj.question or create_chat_obj.question.strip() == '': + raise Exception("Question cannot be Empty") + + chat = Chat(create_time=datetime.datetime.now(), + create_by=current_user.id, + brief=create_chat_obj.question.strip()[:20], + datasource=create_chat_obj.datasource) + + ds = session.query(CoreDatasource).filter(CoreDatasource.id == create_chat_obj.datasource).first() + + if not ds: + raise Exception(f"Datasource with id {create_chat_obj.datasource} not found") + + chat.engine_type = ds.type_name + + chat_info = ChatInfo(**chat.model_dump()) + + session.add(chat) + session.flush() + session.refresh(chat) + chat_info.id = chat.id + session.commit() + + return chat_info + + +def save_question(session: SessionDep, current_user: CurrentUser, question: ChatQuestion) -> ChatRecord: + if not question.chat_id: + raise Exception("ChatId cannot be None") + if not question.question or question.question.strip() == '': + raise Exception("Question cannot be Empty") + + chat = session.query(Chat).filter(Chat.id == question.chat_id).first() + if not chat: + raise Exception(f"Chat with id {question.chat_id} not found") + + record = ChatRecord() + record.question = question.question + record.chat_id = chat.id + record.create_time = datetime.datetime.now() + record.create_by = current_user.id + record.datasource = chat.datasource + record.engine_type = chat.engine_type + + result = ChatRecord(**record.model_dump()) + + session.add(record) + session.flush() + session.refresh(record) + result.id = record.id + session.commit() + + return result + +def save_full_question(session: SessionDep, id: int, full_question: str) -> ChatRecord: + if not id: + raise Exception("Record id cannot be None") + record = session.query(ChatRecord).filter(Chat.id == id).first() + record.full_question = full_question + + result = ChatRecord(**record.model_dump()) + + session.add(record) + session.flush() + session.refresh(record) + + session.commit() + + return result + +def save_answer(session: SessionDep, id: int, answer: str) -> ChatRecord: + if not id: + raise Exception("Record id cannot be None") + + record = session.query(ChatRecord).filter(Chat.id == id).first() + record.answer = answer + + result = ChatRecord(**record.model_dump()) + + session.add(record) + session.flush() + session.refresh(record) + + session.commit() + + return result diff --git a/backend/apps/chat/models/__init__.py b/backend/apps/chat/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/chat/models/chat_model.py b/backend/apps/chat/models/chat_model.py new file mode 100644 index 00000000..c747ae52 --- /dev/null +++ b/backend/apps/chat/models/chat_model.py @@ -0,0 +1,49 @@ +from sqlmodel import SQLModel, Field +from sqlalchemy import Column, Text, BigInteger, DateTime, Integer, Identity +from datetime import datetime +from pydantic import BaseModel +from typing import List, Optional + + +class Chat(SQLModel, table=True): + __tablename__ = "chat" + id: Optional[int] = Field(sa_column=Column(Integer, Identity(always=True), primary_key=True)) + create_time: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=True)) + create_by: int = Field(sa_column=Column(BigInteger, nullable=True)) + brief: str = Field(max_length=64, nullable=True) + chat_type: str = Field(max_length=20, default="chat") # chat, datasource + datasource: int = Field(sa_column=Column(Integer, nullable=False)) + engine_type: str = Field(max_length=64) + + +class ChatRecord(SQLModel, table=True): + __tablename__ = "chat_record" + id: Optional[int] = Field(sa_column=Column(Integer, Identity(always=True), primary_key=True)) + chat_id: int = Field(sa_column=Column(Integer)) + create_time: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=True)) + create_by: int = Field(sa_column=Column(BigInteger, nullable=True)) + datasource: int = Field(sa_column=Column(Integer, nullable=False)) + engine_type: str = Field(max_length=64) + question: str = Field(sa_column=Column(Text, nullable=True)) + full_question: str = Field(sa_column=Column(Text, nullable=True)) + answer: str = Field(sa_column=Column(Text, nullable=True)) + run_time: float = Field(default=0) + + +class CreateChat(BaseModel): + id: int = None + question: str = '' + datasource: int = None + + +class ChatInfo(BaseModel): + id: Optional[int] = None + create_time: datetime = None + create_by: int = None + brief: str = '' + chat_type: str = "chat" + datasource: int = None + engine_type: str = '' + datasource_name: str = '' + datasource_exists: bool = True + records: List[ChatRecord] = [] diff --git a/backend/apps/chat/schemas/chat_schema.py b/backend/apps/chat/schemas/chat_schema.py index 607e5f54..57b4dee7 100644 --- a/backend/apps/chat/schemas/chat_schema.py +++ b/backend/apps/chat/schemas/chat_schema.py @@ -3,4 +3,5 @@ from pydantic import BaseModel class ChatQuestion(BaseModel): - question: str \ No newline at end of file + question: str + chat_id: int \ No newline at end of file diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b33a7151..16956fd8 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "oracledb (>=3.1.1,<4.0.0)" ] [[tool.uv.index]] -url = "https://pypi.tuna.tsinghua.edu.cn/simple" +url = "https://mirrors.aliyun.com/pypi/simple" default = true [tool.uv] diff --git a/frontend/package.json b/frontend/package.json index 4ee98c4a..9ff37c67 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,7 +13,8 @@ "snowflake-id": "^1.1.0", "vue": "^3.5.13", "vue-router": "^4.5.0", - "web-storage-cache": "^1.1.1" + "web-storage-cache": "^1.1.1", + "dayjs": "^1.11.13" }, "devDependencies": { "@element-plus/icons-vue": "^2.3.1", diff --git a/frontend/src/api/chat.ts b/frontend/src/api/chat.ts index 3a5ec56d..91233def 100644 --- a/frontend/src/api/chat.ts +++ b/frontend/src/api/chat.ts @@ -1,15 +1,158 @@ -import { request } from '@/utils/request' +import {request} from '@/utils/request' +import {getDate} from "@/utils/utils.ts"; export const questionApi = { - pager: (pageNumber: number, pageSize: number) => request.get(`/chat/question/pager/${pageNumber}/${pageSize}`), - /* add: (data: any) => new Promise((resolve, reject) => { - request.post('/chat/question', data, { responseType: 'stream', timeout: 0, onDownloadProgress: p => { - resolve(p) - }}).catch(e => reject(e)) - }), */ - // add: (data: any) => request.post('/chat/question', data), - add: (data: any) => request.fetchStream('/chat/question', data), - edit: (data: any) => request.put('/chat/question', data), - delete: (id: number) => request.delete(`/chat/question/${id}`), - query: (id: number) => request.get(`/chat/question/${id}`) + + pager: (pageNumber: number, pageSize: number) => request.get(`/chat/question/pager/${pageNumber}/${pageSize}`), + /* add: (data: any) => new Promise((resolve, reject) => { + request.post('/chat/question', data, { responseType: 'stream', timeout: 0, onDownloadProgress: p => { + resolve(p) + }}).catch(e => reject(e)) + }), */ + // add: (data: any) => request.post('/chat/question', data), + add: (data: any) => request.fetchStream('/chat/question', data), + edit: (data: any) => request.put('/chat/question', data), + delete: (id: number) => request.delete(`/chat/question/${id}`), + query: (id: number) => request.get(`/chat/question/${id}`) +} + +export class ChatRecord { + id?: number + chat_id?: number + create_time?: Date | string + create_by?: number + datasource?: number + engine_type?: string + question?: string + answer?: string + run_time: number = 0 + + constructor() + constructor(id: number, chat_id: number, create_time: Date | string, create_by: number, datasource: number, engine_type: string, question: string, answer: string, run_time: number) + constructor(id?: number, chat_id?: number, create_time?: Date | string, create_by?: number, datasource?: number, engine_type?: string, question?: string, answer?: string, run_time?: number) { + this.id = id + this.chat_id = chat_id + this.create_time = getDate(create_time) + this.create_by = create_by + this.datasource = datasource + this.engine_type = engine_type + this.question = question + this.answer = answer + this.run_time = run_time ?? 0 + } +} + +export class Chat { + id?: number + create_time?: Date | string + create_by?: number + brief?: string + chat_type?: string + datasource?: number + engine_type?: string + + constructor() + constructor(id: number, create_time: Date | string, create_by: number, brief: string, chat_type: string, datasource: number, engine_type: string) + constructor(id?: number, create_time?: Date | string, create_by?: number, brief?: string, chat_type?: string, datasource?: number, engine_type?: string) { + this.id = id + this.create_time = getDate(create_time) + this.create_by = create_by + this.brief = brief + this.chat_type = chat_type + this.datasource = datasource + this.engine_type = engine_type + } +} + +export class ChatInfo extends Chat { + datasource_name?: string + datasource_exists: boolean = true + records: Array = [] + + constructor() + constructor(chat: Chat) + constructor(id: number, create_time: Date | string, create_by: number, brief: string, chat_type: string, datasource: number, engine_type: string, datasource_name: string, datasource_exists: boolean, records: Array) + constructor(param1?: number | Chat, create_time?: Date | string, create_by?: number, brief?: string, chat_type?: string, datasource?: number, engine_type?: string, datasource_name?: string, datasource_exists: boolean = true, records: Array = []) { + super() + if (param1 !== undefined) { + if (param1 instanceof Chat) { + this.id = param1.id + this.create_time = getDate(param1.create_time) + this.create_by = param1.create_by + this.brief = param1.brief + this.chat_type = param1.chat_type + this.datasource = param1.datasource + this.engine_type = param1.engine_type + } else { + this.id = param1 + this.create_time = getDate(create_time) + this.create_by = create_by + this.brief = brief + this.chat_type = chat_type + this.datasource = datasource + this.engine_type = engine_type + } + } + this.datasource_name = datasource_name + this.datasource_exists = datasource_exists + this.records = records + } +} + +function toChatRecord(data?: any): ChatRecord | undefined { + if (!data) { + return undefined + } + return new ChatRecord(data.id, data.number, data.create_time, data.create_by, data.datasource, data.engine_type, data.question, data.answer, data.run_time) +} + +function toChatRecordList(list: any = []): ChatRecord[] { + const records: Array = [] + for (let i = 0; i < list.length; i++) { + const record = toChatRecord(list[i]) + if (record) { + records.push(record) + } + } + return records +} + +function toChatInfo(data?: any): ChatInfo | undefined { + if (!data) { + return undefined + } + return new ChatInfo(data.id, data.create_time, data.create_by, data.brief, data.chat_type, data.datasource, data.engine_type, data.datasource_name, data.datasource_exists, toChatRecordList(data.records)) +} + +function toChatInfoList(list: any[] = []): ChatInfo[] { + const infos: Array = [] + for (let i = 0; i < list.length; i++) { + const chatInfo = toChatInfo(list[i]) + if (chatInfo) { + infos.push(chatInfo) + } + } + return infos +} + +function list(): Promise> { + return request.get('/chat/list') +} + +function get(id: number): Promise { + return request.get(`/chat/get/${id}`) +} + +function startChat(data: any): Promise { + return request.post('/chat/start', data) +} + +export const chatApi = { + toChatRecord, + toChatRecordList, + toChatInfo, + toChatInfoList, + list, + get, + startChat } \ No newline at end of file diff --git a/frontend/src/style.less b/frontend/src/style.less index a0cdd12e..57d36638 100644 --- a/frontend/src/style.less +++ b/frontend/src/style.less @@ -3,9 +3,6 @@ line-height: 1.5; font-weight: 400; - color-scheme: light dark; - color: rgba(255, 255, 255, 0.87); - background-color: #242424; font-synthesis: none; text-rendering: optimizeLegibility; diff --git a/frontend/src/utils/utils.ts b/frontend/src/utils/utils.ts index 59bf7f6b..11aae719 100644 --- a/frontend/src/utils/utils.ts +++ b/frontend/src/utils/utils.ts @@ -1,20 +1,37 @@ +import dayjs from "dayjs"; + const getCheckDate = (timestamp: any) => { - if (!timestamp) return false - const dt = new Date(timestamp) - if (isNaN(dt.getTime())) return false - return dt + if (!timestamp) return false + const dt = new Date(timestamp) + if (isNaN(dt.getTime())) return false + return dt } export const datetimeFormat = (timestamp: any) => { - const dt = getCheckDate(timestamp) - if (!dt) return timestamp + const dt = getCheckDate(timestamp) + if (!dt) return timestamp - const y = dt.getFullYear() - const m = (dt.getMonth() + 1 + '').padStart(2, '0') - const d = (dt.getDate() + '').padStart(2, '0') - const hh = (dt.getHours() + '').padStart(2, '0') - const mm = (dt.getMinutes() + '').padStart(2, '0') - const ss = (dt.getSeconds() + '').padStart(2, '0') + const y = dt.getFullYear() + const m = (dt.getMonth() + 1 + '').padStart(2, '0') + const d = (dt.getDate() + '').padStart(2, '0') + const hh = (dt.getHours() + '').padStart(2, '0') + const mm = (dt.getMinutes() + '').padStart(2, '0') + const ss = (dt.getSeconds() + '').padStart(2, '0') - return `${y}-${m}-${d} ${hh}:${mm}:${ss}` -} \ No newline at end of file + return `${y}-${m}-${d} ${hh}:${mm}:${ss}` +} + +/** + * + * string: only accept ISO 8601, example: '2018-04-04T16:00:00.000Z' + * number: timestamp + * @param time + */ +export function getDate(time?: Date | string | number) { + if (!time) return undefined + if (time instanceof Date) return time + if (typeof time === 'string') { + return dayjs(time).toDate() + } + return new Date(time) +} diff --git a/frontend/src/views/chat/index.vue b/frontend/src/views/chat/index.vue index 2cd6482f..6606dea7 100644 --- a/frontend/src/views/chat/index.vue +++ b/frontend/src/views/chat/index.vue @@ -1,279 +1,288 @@ \ No newline at end of file + diff --git a/frontend/src/views/ds/form.vue b/frontend/src/views/ds/form.vue index 18370e14..df8620e5 100644 --- a/frontend/src/views/ds/form.vue +++ b/frontend/src/views/ds/form.vue @@ -8,17 +8,30 @@ @closed="close" modal-class="add-datasource_dialog" > - - - - -
- + + +
+ @@ -96,7 +109,7 @@
-
+
Test Connect Next - Preview Save(); const emit = defineEmits(["refresh"]); -const active = ref(1); +const active = ref(0); const isCreate = ref(true); const isEditTable = ref(false); const checkList = ref([]); @@ -199,7 +212,7 @@ const config = ref({ const close = () => { dialogVisible.value = false; isCreate.value = true; - active.value = 1; + active.value = 0; isEditTable.value = false; checkList.value = []; tableList.value = []; @@ -230,7 +243,7 @@ const open = (item: any, editTable: boolean = false) => { } if (editTable) { - active.value = 2; + active.value = 1; isEditTable.value = true; isCreate.value = false; // request tables and check tables @@ -421,8 +434,8 @@ defineExpose({ open }); .add-datasource_dialog { .container { max-height: 600px; - margin-top: 20px; - .el-vl__window.el-select-dropdown__list::-webkit-scrollbar { + overflow-y: auto; + .ed-vl__window.ed-select-dropdown__list::-webkit-scrollbar { width: 0; height: 0; } diff --git a/frontend/src/views/ds/index.vue b/frontend/src/views/ds/index.vue index 3e9a57c0..1a2379f1 100644 --- a/frontend/src/views/ds/index.vue +++ b/frontend/src/views/ds/index.vue @@ -65,7 +65,7 @@ import { ref, onMounted } from 'vue' import DsForm from './form.vue' import { datasourceApi } from '@/api/datasource' import { datetimeFormat } from '@/utils/utils' -import { ElMessageBox } from 'element-plus' +import { ElMessageBox } from 'element-plus-secondary' import { useRouter } from 'vue-router' const searchValue = ref('') @@ -148,7 +148,7 @@ const getTables = (id: number, name: string) => { // datasourceApi.execSql(id,'select id,name,table_name from core_dataset_table limit 10').then((res) => { // console.log(res) // }) - router.push(`/ds/dsTable/${id}/${name}`) + router.push(`/dsTable/${id}/${name}`) } onMounted(() => { @@ -263,6 +263,7 @@ onMounted(() => { display: flex; .action-btn { display: none; + min-width: 0; } } :hover .action-btn{ diff --git a/frontend/src/views/login/index.vue b/frontend/src/views/login/index.vue index e2973319..a1e06ece 100644 --- a/frontend/src/views/login/index.vue +++ b/frontend/src/views/login/index.vue @@ -140,7 +140,7 @@ const submitForm = () => { } } -:deep(.el-input__wrapper) { +:deep(.ed-input__wrapper) { background-color: #f5f7fa; } \ No newline at end of file diff --git a/frontend/src/views/setting/Professional.vue b/frontend/src/views/setting/Professional.vue index 73b7136d..3b5f263d 100644 --- a/frontend/src/views/setting/Professional.vue +++ b/frontend/src/views/setting/Professional.vue @@ -262,7 +262,7 @@ onMounted(() => { } .sqlbot-table { border-radius: 6px; - :deep(.el-table) { + :deep(.ed-table) { --el-table-header-bg-color: #f5f7fa; --el-table-border-color: #ebeef5; --el-table-header-text-color: #606266; @@ -303,19 +303,19 @@ onMounted(() => { } .sqlbot-form { width: 100%; - .el-form-item { + .ed-form-item { margin-bottom: 16px; text-align: left; label { font-size: 14px; color: #606266; } - /* .el-input { + /* .ed-input { --el-input-border-color: #e5e8ed; --el-input-border-radius: 4px; --el-input-inner-height: 40px !important; } - .el-select { + .ed-select { --el-select-bg-color: #f5f7fa; --el-select-border-color: #e5e8ed; --el-select-border-radius: 4px; @@ -326,7 +326,7 @@ onMounted(() => { .sqlbot-dialog { width: 500px; - :deep(.el-dialog__header) { + :deep(.ed-dialog__header) { text-align: left !important; font-size: 16px; background-color: #f5f7fa; diff --git a/frontend/src/views/setting/index.vue b/frontend/src/views/setting/index.vue index 404d35c6..432c9dfd 100644 --- a/frontend/src/views/setting/index.vue +++ b/frontend/src/views/setting/index.vue @@ -43,9 +43,9 @@ const handleClick = (tab: any, event: MouseEvent) => { box-shadow: var(--shadow); overflow: hidden; padding: 24px; - :deep(.el-tabs) { + :deep(.ed-tabs) { --el-tabs-header-height: 55px; - .el-tabs__header { + .ed-tabs__header { margin: 0 0 24px !important; } } @@ -63,7 +63,7 @@ const handleClick = (tab: any, event: MouseEvent) => { height: 28px; } } - :deep(.el-tabs__item) { + :deep(.ed-tabs__item) { padding: 0 16px !important; &:hover { // color: var(--primary-color); diff --git a/frontend/src/views/system/model/index.vue b/frontend/src/views/system/model/index.vue index 15979168..41da839d 100644 --- a/frontend/src/views/system/model/index.vue +++ b/frontend/src/views/system/model/index.vue @@ -433,7 +433,7 @@ const batchEnableHandler = () => { } .sqlbot-table { border-radius: 6px; - :deep(.el-table) { + :deep(.ed-table) { --el-table-header-bg-color: #f5f7fa; --el-table-border-color: #ebeef5; --el-table-header-text-color: #606266; @@ -487,7 +487,7 @@ const batchEnableHandler = () => { } .sqlbot-form { width: 100%; - .el-form-item { + .ed-form-item { margin-bottom: 16px; text-align: left; label { @@ -499,7 +499,7 @@ const batchEnableHandler = () => { .sqlbot-dialog { width: 500px; - :deep(.el-dialog__header) { + :deep(.ed-dialog__header) { text-align: left !important; font-size: 16px; background-color: #f5f7fa; diff --git a/frontend/src/views/system/user/index.vue b/frontend/src/views/system/user/index.vue index 1e625710..3a6ca017 100644 --- a/frontend/src/views/system/user/index.vue +++ b/frontend/src/views/system/user/index.vue @@ -315,7 +315,7 @@ onMounted(() => { } .sqlbot-table { border-radius: 6px; - :deep(.el-table) { + :deep(.ed-table) { --el-table-header-bg-color: #f5f7fa; --el-table-border-color: #ebeef5; --el-table-header-text-color: #606266; @@ -369,7 +369,7 @@ onMounted(() => { } .sqlbot-form { width: 100%; - .el-form-item { + .ed-form-item { margin-bottom: 16px; text-align: left; label { @@ -381,7 +381,7 @@ onMounted(() => { .sqlbot-dialog { width: 500px; - :deep(.el-dialog__header) { + :deep(.ed-dialog__header) { text-align: left !important; font-size: 16px; background-color: #f5f7fa; diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 2723d9ad..06526beb 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -1,8 +1,8 @@ import { defineConfig, loadEnv } from 'vite' import vue from '@vitejs/plugin-vue' import AutoImport from 'unplugin-auto-import/vite' -import Components from 'unplugin-vue-components/vite' -import { ElementPlusResolver } from 'unplugin-vue-components/resolvers' +import Components from 'unplugin-vue-components-secondary/vite' +import { ElementPlusResolver } from 'unplugin-vue-components-secondary/resolvers' import path from 'path' import svgLoader from 'vite-svg-loader' export default defineConfig(({ mode }) => { @@ -38,7 +38,7 @@ export default defineConfig(({ mode }) => { rollupOptions: { output: { manualChunks: { - 'element-plus': ['element-plus'], + 'element-plus-secondary': ['element-plus-secondary'], }, }, }, @@ -47,5 +47,15 @@ export default defineConfig(({ mode }) => { jsxFactory: 'h', jsxFragment: 'Fragment', }, + server: { + proxy: { + '/api': { + target: 'http://192.168.11.65:8000', + changeOrigin: true, + rewrite: path => path.replace(/^\/api/, '/api') + } + }, + port: 8080 + } } }) From 689d7adac72fb400170d0668c3694611fe2a06ac Mon Sep 17 00:00:00 2001 From: junjun Date: Fri, 23 May 2025 16:08:11 +0800 Subject: [PATCH 5/6] feat(datasource): front config --- frontend/vite.config.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 06526beb..ed56689d 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -46,16 +46,6 @@ export default defineConfig(({ mode }) => { esbuild: { jsxFactory: 'h', jsxFragment: 'Fragment', - }, - server: { - proxy: { - '/api': { - target: 'http://192.168.11.65:8000', - changeOrigin: true, - rewrite: path => path.replace(/^\/api/, '/api') - } - }, - port: 8080 } } }) From 83d94eedaa8b982cafb61250be786f243122590e Mon Sep 17 00:00:00 2001 From: junjun Date: Fri, 23 May 2025 16:39:29 +0800 Subject: [PATCH 6/6] feat(datasource): provide checked tables and fields --- backend/apps/datasource/api/datasource.py | 6 +++--- backend/apps/datasource/crud/datasource.py | 19 ++++++++++++++----- backend/apps/datasource/crud/field.py | 3 ++- backend/apps/datasource/models/datasource.py | 11 ++++++++++- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/backend/apps/datasource/api/datasource.py b/backend/apps/datasource/api/datasource.py index d50b4061..1eff12f9 100644 --- a/backend/apps/datasource/api/datasource.py +++ b/backend/apps/datasource/api/datasource.py @@ -12,7 +12,7 @@ from ..crud.datasource import get_datasource_list, check_status, create_ds, upda execSql, update_table_and_fields, getTablesByDs, chooseTables, preview from ..crud.field import get_fields_by_table_id from ..crud.table import get_tables_by_ds_id -from ..models.datasource import CoreDatasource, CreateDatasource, EditObj, CoreTable +from ..models.datasource import CoreDatasource, CreateDatasource, TableObj, CoreTable router = APIRouter(tags=["datasource"], prefix="/datasource") path = "/opt/sqlbot/data/excel" @@ -79,12 +79,12 @@ async def field_list(session: SessionDep, id: int): @router.post("/editLocalComment") -async def edit_local(session: SessionDep, data: EditObj): +async def edit_local(session: SessionDep, data: TableObj): update_table_and_fields(session, data) @router.post("/previewData/{id}") -async def edit_local(session: SessionDep, id: int, data: EditObj): +async def edit_local(session: SessionDep, id: int, data: TableObj): return preview(session, id, data) diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index 87a599a6..c15eac4c 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -9,13 +9,13 @@ from apps.datasource.utils.utils import aes_decrypt from apps.db.db import get_engine, get_tables, get_fields, exec_sql from apps.db.engine import get_engine_config from apps.db.engine import get_engine_conn +from apps.db.type import db_type_relation from common.core.deps import SessionDep, CurrentUser from common.utils.utils import deepcopy_ignore_extra from ..crud.field import delete_field_by_ds_id, update_field from ..crud.table import delete_table_by_ds_id, update_table -from ..models.datasource import CoreDatasource, CreateDatasource, CoreTable, CoreField, ColumnSchema, EditObj, \ - DatasourceConf -from apps.db.type import db_type_relation +from ..models.datasource import CoreDatasource, CreateDatasource, CoreTable, CoreField, ColumnSchema, TableObj, \ + DatasourceConf, TableAndFields def get_datasource_list(session: SessionDep): @@ -186,13 +186,13 @@ def sync_fields(session: SessionDep, ds: CoreDatasource, table: CoreTable, field session.commit() -def update_table_and_fields(session: SessionDep, data: EditObj): +def update_table_and_fields(session: SessionDep, data: TableObj): update_table(session, data.table) for field in data.fields: update_field(session, field) -def preview(session: SessionDep, id: int, data: EditObj): +def preview(session: SessionDep, id: int, data: TableObj): ds = session.query(CoreDatasource).filter(CoreDatasource.id == id).first() conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) if ds.type != "excel" else get_engine_config() sql: str = "" @@ -209,3 +209,12 @@ def preview(session: SessionDep, id: int, data: EditObj): ORDER BY "{data.fields[0].field_name}" OFFSET 0 ROWS FETCH NEXT 100 ROWS ONLY""" return exec_sql(ds, sql) + + +def get_tableobj_by_ds(session: SessionDep, id: int) -> List[TableAndFields]: + list: List = [] + tables = session.query(CoreTable).filter(CoreTable.ds_id == id).all() + for table in tables: + fields = session.query(CoreField).filter(and_(CoreField.table_id == table.id, CoreField.checked == True)).all() + list.append(TableAndFields(table=table, fields=fields)) + return list diff --git a/backend/apps/datasource/crud/field.py b/backend/apps/datasource/crud/field.py index eafe3e46..53321575 100644 --- a/backend/apps/datasource/crud/field.py +++ b/backend/apps/datasource/crud/field.py @@ -9,7 +9,8 @@ def delete_field_by_ds_id(session: SessionDep, id: int): def get_fields_by_table_id(session: SessionDep, id: int): - return session.query(CoreField).filter(CoreField.table_id == id).all() + return session.query(CoreField).filter(CoreField.table_id == id).order_by( + CoreField.field_name.asc()).all() def update_field(session: SessionDep, item: CoreField): diff --git a/backend/apps/datasource/models/datasource.py b/backend/apps/datasource/models/datasource.py index 59ef6562..41667d6c 100644 --- a/backend/apps/datasource/models/datasource.py +++ b/backend/apps/datasource/models/datasource.py @@ -55,7 +55,7 @@ class CreateDatasource(BaseModel): # edit local saved table and fields -class EditObj(BaseModel): +class TableObj(BaseModel): table: CoreTable = None fields: List[CoreField] = [] @@ -108,3 +108,12 @@ class ColumnSchema: fieldName: str fieldType: str fieldComment: str + + +class TableAndFields: + def __init__(self, table, fields): + self.table = table + self.fields = fields + + table: CoreTable + fields: List[CoreField]