From 177fc51488292313660520172e6196b0429238aa Mon Sep 17 00:00:00 2001 From: xuwei-fit2cloud Date: Thu, 29 May 2025 10:09:10 +0800 Subject: [PATCH 01/10] Update .gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 237d53ba..3aaaa54c 100644 --- a/.gitignore +++ b/.gitignore @@ -184,4 +184,6 @@ cython_debug/ .pypirc .DS_Store -test.py \ No newline at end of file +test.py + +components.d.ts From 62be1f07b449299496a67a3a55af0308d8774558 Mon Sep 17 00:00:00 2001 From: junjun Date: Thu, 29 May 2025 15:20:05 +0800 Subject: [PATCH 02/10] feat(datasource): form check --- frontend/src/views/ds/form.vue | 134 ++++++++++++++++++++------------- 1 file changed, 81 insertions(+), 53 deletions(-) diff --git a/frontend/src/views/ds/form.vue b/frontend/src/views/ds/form.vue index 30da7e49..a94a9f0e 100644 --- a/frontend/src/views/ds/form.vue +++ b/frontend/src/views/ds/form.vue @@ -38,7 +38,7 @@ - +
- - + + - - + + - + - + - - + + - - + + Service Name SID - + - - + + Get Schema @@ -189,6 +189,24 @@ const rules = reactive({ { required: true, message: "Please input name", trigger: "blur" }, { min: 1, max: 50, message: "Length should be 1 to 50", trigger: "blur" }, ], + type: [ + { required: true, message: "Please choose database type" ,trigger: "change"} + ], + host: [ + { required: true, message: "Please input host" ,trigger: "blur"} + ], + port: [ + { required: true, message: "Please input port" ,trigger: "blur"} + ], + database: [ + { required: true, message: "Please input database" ,trigger: "blur"} + ], + mode: [ + { required: true, message: "Please choose mode" ,trigger: "change"} + ], + dbSchema: [ + { required: true, message: "Please input schema" ,trigger: "blur"} + ] }); const dialogVisible = ref(false); @@ -197,8 +215,6 @@ const form = ref({ description: "", type: "mysql", configuration: "", -}); -const config = ref({ driver: "", host: "", port: 0, @@ -234,16 +250,16 @@ const open = (item: any, editTable: boolean = false) => { form.value.configuration = item.configuration; if (item.configuration) { const configuration = JSON.parse(decrypted(item.configuration)); - config.value.host = configuration.host; - config.value.port = configuration.port; - config.value.username = configuration.username; - config.value.password = configuration.password; - config.value.database = configuration.database; - config.value.extraJdbc = configuration.extraJdbc; - config.value.dbSchema = configuration.dbSchema; - config.value.filename = configuration.filename; - config.value.sheets = configuration.sheets; - config.value.mode = configuration.mode; + form.value.host = configuration.host; + form.value.port = configuration.port; + form.value.username = configuration.username; + form.value.password = configuration.password; + form.value.database = configuration.database; + form.value.extraJdbc = configuration.extraJdbc; + form.value.dbSchema = configuration.dbSchema; + form.value.filename = configuration.filename; + form.value.sheets = configuration.sheets; + form.value.mode = configuration.mode; } if (editTable) { @@ -258,11 +274,12 @@ const open = (item: any, editTable: boolean = false) => { return ele.table_name; }); if (item.type === "excel") { - tableList.value = config.value.sheets; + tableList.value = form.value.sheets; } else { tableListLoading.value = true; + const requestObj = buildConf() datasourceApi - .getTablesByConf(form.value) + .getTablesByConf(requestObj) .then((table) => { tableList.value = table; checkList.value = checkList.value.filter((ele: string) => { @@ -290,8 +307,6 @@ const open = (item: any, editTable: boolean = false) => { description: "", type: "mysql", configuration: "", - }; - config.value = { driver: "", host: "", port: 0, @@ -303,7 +318,7 @@ const open = (item: any, editTable: boolean = false) => { filename: "", sheets: [], mode: "service_name", - }; + } } dialogVisible.value = true; }; @@ -320,11 +335,11 @@ const save = async (formEl: FormInstance | undefined) => { return { table_name: ele.tableName, table_comment: ele.tableComment }; }); - buildConf(); + const requestObj = buildConf(); if (form.value.id) { if (!isEditTable.value) { // only update datasource config info - datasourceApi.update(form.value).then((res) => { + datasourceApi.update(requestObj).then((res) => { console.log(res); close(); emit("refresh"); @@ -336,8 +351,8 @@ const save = async (formEl: FormInstance | undefined) => { }); } } else { - form.value.tables = list; - datasourceApi.add(form.value).then((res: any) => { + requestObj.tables = list; + datasourceApi.add(requestObj).then((res: any) => { console.log(res); close(); emit("refresh"); @@ -350,23 +365,36 @@ const save = async (formEl: FormInstance | undefined) => { const buildConf = () => { form.value.configuration = encrypted( JSON.stringify({ - host: config.value.host, - port: config.value.port, - username: config.value.username, - password: config.value.password, - database: config.value.database, - extraJdbc: config.value.extraJdbc, - dbSchema: config.value.dbSchema, - filename: config.value.filename, - sheets: config.value.sheets, - mode: config.value.mode, + host: form.value.host, + port: form.value.port, + username: form.value.username, + password: form.value.password, + database: form.value.database, + extraJdbc: form.value.extraJdbc, + dbSchema: form.value.dbSchema, + filename: form.value.filename, + sheets: form.value.sheets, + mode: form.value.mode, }) - ); + ) + const obj = JSON.parse(JSON.stringify(form.value)) + delete obj.driver + delete obj.host + delete obj.port + delete obj.username + delete obj.password + delete obj.database + delete obj.extraJdbc + delete obj.dbSchema + delete obj.filename + delete obj.sheets + delete obj.mode + return obj }; const check = () => { - buildConf(); - datasourceApi.check(form.value).then((res: any) => { + const requestObj = buildConf(); + datasourceApi.check(requestObj).then((res: any) => { if (res) { ElMessage({ message: "Connect success", @@ -394,12 +422,12 @@ const next = async (formEl: FormInstance | undefined) => { } } else { // check status if success do next - buildConf(); - datasourceApi.check(form.value).then((res: boolean) => { + const requestObj = buildConf(); + datasourceApi.check(requestObj).then((res: boolean) => { if (res) { active.value++; // request tables - datasourceApi.getTablesByConf(form.value).then((res) => { + datasourceApi.getTablesByConf(requestObj).then((res) => { tableList.value = res; }); } else { @@ -428,8 +456,8 @@ const beforeUpload = (rawFile: any) => { }; const onSuccess = (response: any) => { - config.value.filename = response.data.filename; - config.value.sheets = response.data.sheets; + form.value.filename = response.data.filename; + form.value.sheets = response.data.sheets; tableList.value = response.data.sheets; excelUploadSuccess.value = true; }; From d833273e4009445eb9bb19988cc8e574334b20cd Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Thu, 29 May 2025 15:30:45 +0800 Subject: [PATCH 03/10] perf: Auto exec alembic --- backend/main.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/backend/main.py b/backend/main.py index f8858542..2cb6b8d3 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,4 +1,5 @@ import logging +from fastapi.concurrency import asynccontextmanager from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles import os @@ -11,6 +12,16 @@ from apps.api import api_router from apps.system.middleware.auth import TokenMiddleware from common.core.config import settings from common.core.response_middleware import ResponseMiddleware, exception_handler +from alembic.config import Config +from alembic import command + +def run_migrations(): + alembic_cfg = Config("alembic.ini") + command.upgrade(alembic_cfg, "head") +@asynccontextmanager +async def lifespan(app: FastAPI): + run_migrations() + yield def custom_generate_unique_id(route: APIRoute) -> str: tag = route.tags[0] if route.tags and len(route.tags) > 0 else "" @@ -24,6 +35,7 @@ app = FastAPI( title=settings.PROJECT_NAME, openapi_url=f"{settings.API_V1_STR}/openapi.json", generate_unique_id_function=custom_generate_unique_id, + lifespan=lifespan ) # Set all CORS enabled origins From 900caf29d060cb9aa9e6564e1b8514cf4dd95308 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Thu, 29 May 2025 16:44:08 +0800 Subject: [PATCH 04/10] perf: OpenAI is used by default --- frontend/src/entity/CommonEntity.ts | 4 ++-- frontend/src/views/system/model/index.vue | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/src/entity/CommonEntity.ts b/frontend/src/entity/CommonEntity.ts index d926132d..d7db8783 100644 --- a/frontend/src/entity/CommonEntity.ts +++ b/frontend/src/entity/CommonEntity.ts @@ -6,13 +6,13 @@ export interface SelectOption { export const modelTypeOptions: SelectOption[] = [ { label: 'OpenAI', value: 0 }, - { label: 'Anthropic', value: 1 }, + /* { label: 'Anthropic', value: 1 }, { label: 'Baidu', value: 2 }, { label: 'iFLYTEK', value: 3 }, { label: 'Zhipu AI', value: 4 }, { label: 'MiniMax', value: 5 }, { label: 'Tencent', value: 6 }, - { label: 'Other', value: 7 }, + { label: 'Other', value: 7 }, */ ] export const getModelTypeName = (value: any) => { diff --git a/frontend/src/views/system/model/index.vue b/frontend/src/views/system/model/index.vue index 41da839d..72789b8e 100644 --- a/frontend/src/views/system/model/index.vue +++ b/frontend/src/views/system/model/index.vue @@ -191,6 +191,7 @@ const handleSearch = (e: any) => { const editHandler = (id: any) => { modelApi.query(id).then((res: any) => { state.form = res + state.form.temperature = state.form.temperature * 100 dialogFormVisible.value = true }) } @@ -259,6 +260,7 @@ const addTerm = () => { ...state.form, } delete param.id + param.temperature = param.temperature / 100 modelApi.add(param).then(() => { dialogFormVisible.value = false search() @@ -269,6 +271,8 @@ const addTerm = () => { }) } const editTerm = () => { + const param = state.form + param.temperature = param.temperature / 100 modelApi.edit(state.form).then((res: any) => { console.log('edit term', res) dialogFormVisible.value = false From f51f4ce7fca8b4e5f520855b9ce1519181ce927e Mon Sep 17 00:00:00 2001 From: xuwei-fit2cloud Date: Thu, 29 May 2025 17:04:04 +0800 Subject: [PATCH 05/10] refactor: optimize Dockerfile --- .dockerignore | 12 +++++++++ .gitignore | 2 +- Dockerfile | 62 +++++++++++++++++++++++++++++++++++++++++++ backend/.dockerignore | 8 ------ backend/Dockerfile | 52 ------------------------------------ 5 files changed, 75 insertions(+), 61 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile delete mode 100644 backend/.dockerignore delete mode 100644 backend/Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..6b78531c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +# Python +backend/__pycache__ +backend/app.egg-info +backend/*.pyc +backend/.mypy_cache +backend/.coverage +backend/htmlcov +backend/.venv +backend/.gitignore +backend/README.md +backend/scripts +backend/.DS_Store \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3aaaa54c..576b2752 100644 --- a/.gitignore +++ b/.gitignore @@ -186,4 +186,4 @@ cython_debug/ .DS_Store test.py -components.d.ts + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..5ccc1471 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,62 @@ +# Build stage +FROM python:3.11-slim-bookworm AS builder + +# Set build environment variables +ENV PYTHONUNBUFFERED=1 +ENV SQLBOT_HOME=/opt/sqlbot +ENV APP_HOME=${SQLBOT_HOME}/app +ENV UI_HOME=${SQLBOT_HOME}/frontend +ENV PYTHONPATH=${SQLBOT_HOME}/app +ENV PATH="${APP_HOME}/.venv/bin:$PATH" +ENV UV_COMPILE_BYTECODE=1 +ENV UV_LINK_MODE=copy +ENV DEBIAN_FRONTEND=noninteractive + +# Create necessary directories +RUN mkdir -p ${APP_HOME} ${UI_HOME} + +WORKDIR ${APP_HOME} + +# Install uv tool +COPY --from=ghcr.io/astral-sh/uv:0.7.8 /uv /uvx /bin/ + +# COPY ../frontend/dist /opt/sqlbot/frontend/dist +COPY frontend/dist ${UI_HOME}/dist + +# Install dependencies +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=backend/uv.lock,target=uv.lock \ + --mount=type=bind,source=backend/pyproject.toml,target=pyproject.toml \ + uv sync --frozen --no-install-project + +COPY ./backend ${APP_HOME} + +# Final sync to ensure all dependencies are installed +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync + +# Runtime stage +FROM python:3.11-slim-bookworm + +# Set runtime environment variables +ENV PYTHONUNBUFFERED=1 +ENV SQLBOT_HOME=/opt/sqlbot +ENV PYTHONPATH=${SQLBOT_HOME}/app +ENV PATH="${SQLBOT_HOME}/app/.venv/bin:$PATH" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Copy necessary files from builder +COPY --from=builder ${SQLBOT_HOME} ${SQLBOT_HOME} + +WORKDIR ${SQLBOT_HOME}/app + +# Add health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:8000 || exit 1 + +# Run with uvicorn +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "4", "--proxy-headers"] \ No newline at end of file diff --git a/backend/.dockerignore b/backend/.dockerignore deleted file mode 100644 index c0de4abf..00000000 --- a/backend/.dockerignore +++ /dev/null @@ -1,8 +0,0 @@ -# Python -__pycache__ -app.egg-info -*.pyc -.mypy_cache -.coverage -htmlcov -.venv diff --git a/backend/Dockerfile b/backend/Dockerfile deleted file mode 100644 index d1fe0fdc..00000000 --- a/backend/Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -FROM python:3.11-slim-bookworm - -ENV PYTHONUNBUFFERED=1 - - -ENV SQLBOT_HOME=/opt/sqlbot -ENV APP_HOME=$SQLBOT_HOME/app -ENV UI_HOME=$SQLBOT_HOME/frontend -RUN mkdir -p $APP_HOME -RUN mkdir -p $UI_HOME - - - - -WORKDIR $APP_HOME -ENV PYTHONPATH=$APP_HOME - -# COPY ../frontend/dist $UI_HOME/dist -COPY ./dist $UI_HOME/dist -# Install uv -# Ref: https://docs.astral.sh/uv/guides/integration/docker/#installing-uv -COPY --from=ghcr.io/astral-sh/uv:0.5.11 /uv /uvx /bin/ - -# Place executables in the environment at the front of the path -# Ref: https://docs.astral.sh/uv/guides/integration/docker/#using-the-environment -ENV PATH="$APP_HOME/.venv/bin:$PATH" - -# Compile bytecode -# Ref: https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode -ENV UV_COMPILE_BYTECODE=1 - -# uv Cache -# Ref: https://docs.astral.sh/uv/guides/integration/docker/#caching -ENV UV_LINK_MODE=copy - -# Install dependencies -# Ref: https://docs.astral.sh/uv/guides/integration/docker/#intermediate-layers -RUN test -f "./uv.lock" && \ - --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,source=uv.lock,target=uv.lock \ - --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - uv sync --frozen --no-install-project || echo "uv.lock file not found, skipping intermediate-layers" - -COPY . $APP_HOME - - -# Sync the project -# Ref: https://docs.astral.sh/uv/guides/integration/docker/#intermediate-layers -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync - -CMD ["fastapi", "run", "--workers", "4", "main.py"] From 00053a2a382bfa85343bb2f644b7086528f80cfa Mon Sep 17 00:00:00 2001 From: xuwei-fit2cloud Date: Thu, 29 May 2025 17:05:49 +0800 Subject: [PATCH 06/10] refactor: update build docker action --- .github/workflows/build_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index ed9b37ce..6d0b12d2 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -48,7 +48,7 @@ jobs: - name: Build and push Docker image uses: docker/build-push-action@v4 with: - context: ./backend + context: . push: true tags: | ${{ secrets.DOCKERHUB_REGISTRY }}/dataease-v2/sqlbot:latest From 0f19962796d7c7da34243d5fc1e804a549e6f3ba Mon Sep 17 00:00:00 2001 From: xuwei-fit2cloud Date: Thu, 29 May 2025 18:38:36 +0800 Subject: [PATCH 07/10] refactor: update base image --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5ccc1471..3593ad4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build stage -FROM python:3.11-slim-bookworm AS builder +FROM registry.cn-qingdao.aliyuncs.com/dataease/python:3.11-slim-bookworm AS builder # Set build environment variables ENV PYTHONUNBUFFERED=1 @@ -36,7 +36,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv sync # Runtime stage -FROM python:3.11-slim-bookworm +FROM registry.cn-qingdao.aliyuncs.com/dataease/python:3.11-slim-bookworm # Set runtime environment variables ENV PYTHONUNBUFFERED=1 From 8cbad5ac9337cbe598cfd2fbc174608708df76c8 Mon Sep 17 00:00:00 2001 From: xuwei-fit2cloud Date: Thu, 29 May 2025 18:43:54 +0800 Subject: [PATCH 08/10] fix: check uv.lock --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3593ad4a..f39b8607 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,8 @@ COPY --from=ghcr.io/astral-sh/uv:0.7.8 /uv /uvx /bin/ COPY frontend/dist ${UI_HOME}/dist # Install dependencies -RUN --mount=type=cache,target=/root/.cache/uv \ +RUN test -f "./uv.lock" && \ + --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=backend/uv.lock,target=uv.lock \ --mount=type=bind,source=backend/pyproject.toml,target=pyproject.toml \ uv sync --frozen --no-install-project From 5de9b7d330c9edbc5e166be76f7e5b853e95fe0f Mon Sep 17 00:00:00 2001 From: xuwei-fit2cloud Date: Thu, 29 May 2025 18:52:04 +0800 Subject: [PATCH 09/10] chore: update uv source --- Dockerfile | 2 +- backend/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f39b8607..314e064b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ RUN test -f "./uv.lock" && \ --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=backend/uv.lock,target=uv.lock \ --mount=type=bind,source=backend/pyproject.toml,target=pyproject.toml \ - uv sync --frozen --no-install-project + uv sync --frozen --no-install-project || echo "uv.lock file not found, skipping intermediate-layers" COPY ./backend ${APP_HOME} diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 192d20b3..f1f911ac 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "oracledb (>=3.1.1,<4.0.0)" ] [[tool.uv.index]] -url = "https://mirrors.aliyun.com/pypi/simple" +url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" default = true [tool.uv] From 054296741cd52545f78b9afe95792423501e16b2 Mon Sep 17 00:00:00 2001 From: ulleo Date: Fri, 30 May 2025 17:41:45 +0800 Subject: [PATCH 10/10] feat: generate sql with history --- .../versions/1f077c30e476_009_modify_chat.py | 64 ++++++ backend/apps/ai_model/__init__.py | 0 backend/apps/ai_model/llm.py | 1 + .../model_factory.py} | 18 +- backend/apps/ai_model/openai/__init__.py | 0 backend/apps/ai_model/openai/llm.py | 1 + backend/apps/chat/api/chat.py | 148 +++++------- backend/apps/chat/curd/chat.py | 39 +++- backend/apps/chat/models/chat_model.py | 41 +++- backend/apps/chat/schemas/chat_schema.py | 7 - backend/apps/chat/schemas/llm.py | 197 ---------------- backend/apps/chat/task/__init__.py | 0 backend/apps/chat/task/llm.py | 210 ++++++++++++++++++ backend/apps/datasource/crud/datasource.py | 29 +++ backend/apps/template/__init__.py | 0 .../template/generate_analysis/__init__.py | 0 .../apps/template/generate_chart/__init__.py | 0 .../apps/template/generate_chart/generator.py | 6 + .../apps/template/generate_sql/__init__.py | 0 .../apps/template/generate_sql/generator.py | 6 + backend/apps/template/template.py | 15 ++ backend/pyproject.toml | 3 +- backend/template.yaml | 52 +++++ frontend/components.d.ts | 51 ----- frontend/src/views/system/model/index.vue | 4 +- 25 files changed, 526 insertions(+), 366 deletions(-) create mode 100644 backend/alembic/versions/1f077c30e476_009_modify_chat.py create mode 100644 backend/apps/ai_model/__init__.py create mode 100644 backend/apps/ai_model/llm.py rename backend/apps/{chat/schemas/chat_base_schema.py => ai_model/model_factory.py} (86%) create mode 100644 backend/apps/ai_model/openai/__init__.py create mode 100644 backend/apps/ai_model/openai/llm.py delete mode 100644 backend/apps/chat/schemas/chat_schema.py delete mode 100644 backend/apps/chat/schemas/llm.py create mode 100644 backend/apps/chat/task/__init__.py create mode 100644 backend/apps/chat/task/llm.py create mode 100644 backend/apps/template/__init__.py create mode 100644 backend/apps/template/generate_analysis/__init__.py create mode 100644 backend/apps/template/generate_chart/__init__.py create mode 100644 backend/apps/template/generate_chart/generator.py create mode 100644 backend/apps/template/generate_sql/__init__.py create mode 100644 backend/apps/template/generate_sql/generator.py create mode 100644 backend/apps/template/template.py create mode 100644 backend/template.yaml delete mode 100644 frontend/components.d.ts diff --git a/backend/alembic/versions/1f077c30e476_009_modify_chat.py b/backend/alembic/versions/1f077c30e476_009_modify_chat.py new file mode 100644 index 00000000..8db7b9d9 --- /dev/null +++ b/backend/alembic/versions/1f077c30e476_009_modify_chat.py @@ -0,0 +1,64 @@ +"""009_modify_chat + +Revision ID: 1f077c30e476 +Revises: 35d925df4568 +Create Date: 2025-05-30 16:11:08.020715 + +""" +from alembic import op +import sqlalchemy as sa +import sqlmodel.sql.sqltypes +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '1f077c30e476' +down_revision = '35d925df4568' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('chat_record') + op.create_table('chat_record', + sa.Column('id', sa.Integer(), sa.Identity(always=True), nullable=False), + sa.Column('chat_id', sa.Integer(), nullable=True), + sa.Column('create_time', sa.DateTime(timezone=True), nullable=True), + sa.Column('finish_time', sa.DateTime(timezone=True), nullable=True), + sa.Column('create_by', sa.BigInteger(), nullable=True), + sa.Column('datasource', sa.Integer(), nullable=False), + sa.Column('engine_type', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False), + sa.Column('question', sa.Text(), nullable=True), + sa.Column('sql_answer', sa.Text(), nullable=True), + sa.Column('sql', sa.Text(), nullable=True), + sa.Column('sql_exec_result', sa.Text(), nullable=True), + sa.Column('data', sa.Text(), nullable=True), + sa.Column('chart_answer', sa.Text(), nullable=True), + sa.Column('chart', sa.Text(), nullable=True), + sa.Column('full_sql_message', sa.Text(), nullable=True), + sa.Column('full_chart_message', sa.Text(), nullable=True), + sa.Column('finish', sa.Boolean(), nullable=True), + sa.Column('error', sa.Text(), nullable=True), + sa.Column('run_time', sa.Float(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('chat_record') + op.create_table('chat_record', + sa.Column('id', sa.Integer(), sa.Identity(always=True), nullable=False), + sa.Column('chat_id', sa.Integer(), nullable=True), + sa.Column('create_time', sa.DateTime(timezone=True), nullable=True), + sa.Column('create_by', sa.BigInteger(), nullable=True), + sa.Column('datasource', sa.Integer(), nullable=False), + sa.Column('engine_type', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False), + sa.Column('question', sa.Text(), nullable=True), + sa.Column('full_question', sa.Text(), nullable=True), + sa.Column('answer', sa.Text(), nullable=True), + sa.Column('run_time', sa.Float(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### diff --git a/backend/apps/ai_model/__init__.py b/backend/apps/ai_model/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/ai_model/llm.py b/backend/apps/ai_model/llm.py new file mode 100644 index 00000000..99f10893 --- /dev/null +++ b/backend/apps/ai_model/llm.py @@ -0,0 +1 @@ +# todo \ No newline at end of file diff --git a/backend/apps/chat/schemas/chat_base_schema.py b/backend/apps/ai_model/model_factory.py similarity index 86% rename from backend/apps/chat/schemas/chat_base_schema.py rename to backend/apps/ai_model/model_factory.py index 691bd010..b9bcc908 100644 --- a/backend/apps/chat/schemas/chat_base_schema.py +++ b/backend/apps/ai_model/model_factory.py @@ -3,6 +3,10 @@ from typing import Optional, Dict, Any, Type from abc import ABC, abstractmethod from langchain_core.language_models import BaseLLM as LangchainBaseLLM from langchain_openai import ChatOpenAI + +from apps.system.models.system_model import AiModelDetail + + # from langchain_community.llms import Tongyi, VLLM class LLMConfig(BaseModel): @@ -84,4 +88,16 @@ class LLMFactory: @classmethod def register_llm(cls, model_type: str, llm_class: Type[BaseLLM]): """Register new model type""" - cls._llm_types[model_type] = llm_class \ No newline at end of file + cls._llm_types[model_type] = llm_class + + +# todo +def get_llm_config(aimodel: AiModelDetail) -> LLMConfig: + config = LLMConfig( + model_type="openai", + model_name=aimodel.name, + api_key=aimodel.api_key, + api_base_url=aimodel.endpoint, + additional_params={"temperature": aimodel.temperature} + ) + return config \ No newline at end of file diff --git a/backend/apps/ai_model/openai/__init__.py b/backend/apps/ai_model/openai/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/ai_model/openai/llm.py b/backend/apps/ai_model/openai/llm.py new file mode 100644 index 00000000..044a4824 --- /dev/null +++ b/backend/apps/ai_model/openai/llm.py @@ -0,0 +1 @@ +# todo diff --git a/backend/apps/chat/api/chat.py b/backend/apps/chat/api/chat.py index 4f4a7087..0b26ea14 100644 --- a/backend/apps/chat/api/chat.py +++ b/backend/apps/chat/api/chat.py @@ -1,19 +1,19 @@ +import asyncio +import json +from typing import List + from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse from sqlmodel import select from apps.chat.curd.chat import list_chats, get_chat_with_records, create_chat, save_question, save_answer, rename_chat, \ - delete_chat -from apps.chat.models.chat_model import CreateChat, ChatRecord, RenameChat, Chat -from apps.chat.schemas.chat_base_schema import LLMConfig -from apps.chat.schemas.chat_schema import ChatQuestion -from apps.chat.schemas.llm import AgentService -from apps.datasource.crud.datasource import get_table_obj_by_ds + delete_chat, list_records +from apps.chat.models.chat_model import CreateChat, ChatRecord, RenameChat, Chat, ChatQuestion +from apps.chat.task.llm import LLMService +from apps.datasource.crud.datasource import get_table_schema from apps.datasource.models.datasource import CoreDatasource from apps.system.models.system_model import AiModelDetail from common.core.deps import SessionDep, CurrentUser -import json -import asyncio router = APIRouter(tags=["Data Q&A"], prefix="/chat") @@ -96,14 +96,7 @@ async def stream_sql(session: SessionDep, current_user: CurrentUser, request_que detail="No available datasource configuration found" ) - record: ChatRecord - try: - record = save_question(session=session, current_user=current_user, question=request_question) - except Exception as e1: - raise HTTPException( - status_code=400, - detail=str(e1) - ) + request_question.engine = ds.type_name # Get available AI model aimodel = session.exec(select(AiModelDetail).where( @@ -112,93 +105,54 @@ async def stream_sql(session: SessionDep, current_user: CurrentUser, request_que )).first() if not aimodel: raise HTTPException( - status_code=400, + status_code=500, detail="No available AI model configuration found" ) - # Use Tongyi Qianwen - tongyi_config = LLMConfig( - model_type="openai", - model_name=aimodel.name, - api_key=aimodel.api_key, - api_base_url=aimodel.endpoint, - additional_params={"temperature": aimodel.temperature} - ) - # llm_service = LLMService(tongyi_config) - llm_service = AgentService(tongyi_config, ds) - - # Use Custom VLLM model - """ vllm_config = LLMConfig( - model_type="vllm", - model_name="your-model-path", - additional_params={ - "max_new_tokens": 200, - "temperature": 0.3 - } - ) - vllm_service = LLMService(vllm_config) """ - """ result = llm_service.generate_sql(question) - return result """ - + history_records: List[ChatRecord] = list_records(session=session, current_user=current_user, + chart_id=request_question.chat_id) # get schema - schema_str = "" - table_objs = get_table_obj_by_ds(session=session, ds=ds) - db_name = table_objs[0].schema - schema_str += f"【DB_ID】 {db_name}\n【Schema】\n" - for obj in table_objs: - schema_str += f"# Table: {db_name}.{obj.table.table_name}" - table_comment = '' - if obj.table.custom_comment: - table_comment = obj.table.custom_comment.strip() - if table_comment == '': - schema_str += '\n[\n' - else: - schema_str += f", {table_comment}\n[\n" + request_question.db_schema = get_table_schema(session=session, ds=ds) + llm_service = LLMService(request_question, history_records, ds, aimodel) - field_list = [] - for field in obj.fields: - field_comment = '' - if field.custom_comment: - field_comment = field.custom_comment.strip() - if field_comment == '': - field_list.append(f"({field.field_name}:{field.field_type})") - else: - field_list.append(f"({field.field_name}:{field.field_type}, {field_comment})") - schema_str += ",\n".join(field_list) - schema_str += '\n]\n' + llm_service.init_record(session=session, current_user=current_user) - print(schema_str) + def run_task(): + sql_res = llm_service.generate_sql(session=session) + for chunk in sql_res: + yield json.dumps({'content': chunk, 'type': 'sql'}) + '\n\n' + yield json.dumps({'type': 'info', 'msg': 'sql generated'}) + '\n\n' - async def event_generator(): - all_text = '' - try: - async for chunk in llm_service.async_generate(question, schema_str): - data = json.loads(chunk.replace('data: ', '')) - - if data['type'] in ['final', 'tool_result']: - content = data['content'] - print('-- ' + content) - all_text += content - for char in content: - yield f"data: {json.dumps({'type': 'char', 'content': char})}\n\n" - await asyncio.sleep(0.05) - - if 'html' in data: - yield f"data: {json.dumps({'type': 'html', 'content': data['html']})}\n\n" - else: - yield chunk - - except Exception as e: - all_text = 'Exception:' + str(e) - yield f"data: {json.dumps({'type': 'error', 'content': str(e)})}\n\n" - - try: - save_answer(session=session, id=record.id, answer=all_text) - except Exception as e: - raise HTTPException( - status_code=500, - detail=str(e) - ) + # async def event_generator(): + # all_text = '' + # try: + # async for chunk in llm_service.async_generate(question, request_question.db_schema): + # data = json.loads(chunk.replace('data: ', '')) + # + # if data['type'] in ['final', 'tool_result']: + # content = data['content'] + # print('-- ' + content) + # all_text += content + # for char in content: + # yield f"data: {json.dumps({'type': 'char', 'content': char})}\n\n" + # await asyncio.sleep(0.05) + # + # if 'html' in data: + # yield f"data: {json.dumps({'type': 'html', 'content': data['html']})}\n\n" + # else: + # yield chunk + # + # except Exception as e: + # all_text = 'Exception:' + str(e) + # yield f"data: {json.dumps({'type': 'error', 'content': str(e)})}\n\n" + # + # try: + # save_answer(session=session, id=record.id, answer=all_text) + # except Exception as e: + # raise HTTPException( + # status_code=500, + # detail=str(e) + # ) # return EventSourceResponse(event_generator(), headers={"Content-Type": "text/event-stream"}) - return StreamingResponse(event_generator(), media_type="text/event-stream") + return StreamingResponse(run_task(), media_type="text/event-stream") diff --git a/backend/apps/chat/curd/chat.py b/backend/apps/chat/curd/chat.py index 5f6c2658..5ba0437e 100644 --- a/backend/apps/chat/curd/chat.py +++ b/backend/apps/chat/curd/chat.py @@ -1,12 +1,9 @@ import datetime -import json from typing import List -from sqlalchemy import text, and_ -from sqlmodel import select +from sqlalchemy import and_ -from apps.chat.models.chat_model import Chat, ChatRecord, CreateChat, ChatInfo, RenameChat -from apps.chat.schemas.chat_schema import ChatQuestion +from apps.chat.models.chat_model import Chat, ChatRecord, CreateChat, ChatInfo, RenameChat, ChatQuestion from apps.datasource.models.datasource import CoreDatasource from common.core.deps import SessionDep, CurrentUser @@ -66,6 +63,12 @@ def get_chat_with_records(session: SessionDep, chart_id: int, current_user: Curr return chat_info +def list_records(session: SessionDep, chart_id: int, current_user: CurrentUser) -> List[ChatRecord]: + record_list = session.query(ChatRecord).filter( + and_(Chat.create_by == current_user.id, ChatRecord.chat_id == chart_id)).order_by(ChatRecord.create_time).all() + return record_list + + def create_chat(session: SessionDep, current_user: CurrentUser, create_chat_obj: CreateChat) -> ChatInfo: if not create_chat_obj.datasource: raise Exception("Datasource cannot be None") @@ -125,11 +128,33 @@ def save_question(session: SessionDep, current_user: CurrentUser, question: Chat return result -def save_full_question(session: SessionDep, id: int, full_question: str) -> ChatRecord: +def save_full_sql_message(session: SessionDep, record_id: int, full_message: str) -> ChatRecord: + return save_full_sql_message_and_answer(session=session, record_id=record_id, full_message=full_message, answer='') + + +def save_full_sql_message_and_answer(session: SessionDep, record_id: int, answer: str, full_message: str) -> ChatRecord: + if not record_id: + raise Exception("Record id cannot be None") + record = session.query(ChatRecord).filter(ChatRecord.id == record_id).first() + record.full_sql_message = full_message + record.sql_answer = answer + + result = ChatRecord(**record.model_dump()) + + session.add(record) + session.flush() + session.refresh(record) + + session.commit() + + return result + + +def save_full_chart_message(session: SessionDep, id: int, full_message: str) -> ChatRecord: if not id: raise Exception("Record id cannot be None") record = session.query(ChatRecord).filter(ChatRecord.id == id).first() - record.full_question = full_question + record.full_chart_message = full_message result = ChatRecord(**record.model_dump()) diff --git a/backend/apps/chat/models/chat_model.py b/backend/apps/chat/models/chat_model.py index 1c276caa..b65af0ad 100644 --- a/backend/apps/chat/models/chat_model.py +++ b/backend/apps/chat/models/chat_model.py @@ -1,9 +1,12 @@ from sqlmodel import SQLModel, Field -from sqlalchemy import Column, Text, BigInteger, DateTime, Integer, Identity +from sqlalchemy import Column, Text, BigInteger, DateTime, Integer, Identity, Boolean from datetime import datetime from pydantic import BaseModel from typing import List, Optional +from apps.template.generate_chart.generator import get_chart_template +from apps.template.generate_sql.generator import get_sql_template + class Chat(SQLModel, table=True): __tablename__ = "chat" @@ -21,12 +24,21 @@ class ChatRecord(SQLModel, table=True): id: Optional[int] = Field(sa_column=Column(Integer, Identity(always=True), primary_key=True)) chat_id: int = Field(sa_column=Column(Integer)) create_time: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=True)) + finish_time: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=True)) create_by: int = Field(sa_column=Column(BigInteger, nullable=True)) datasource: int = Field(sa_column=Column(Integer, nullable=False)) engine_type: str = Field(max_length=64) question: str = Field(sa_column=Column(Text, nullable=True)) - full_question: str = Field(sa_column=Column(Text, nullable=True)) - answer: str = Field(sa_column=Column(Text, nullable=True)) + sql_answer: str = Field(sa_column=Column(Text, nullable=True)) + sql: str = Field(sa_column=Column(Text, nullable=True)) + sql_exec_result: str = Field(sa_column=Column(Text, nullable=True)) + data: str = Field(sa_column=Column(Text, nullable=True)) + chart_answer: str = Field(sa_column=Column(Text, nullable=True)) + chart: str = Field(sa_column=Column(Text, nullable=True)) + full_sql_message: str = Field(sa_column=Column(Text, nullable=True)) + full_chart_message: str = Field(sa_column=Column(Text, nullable=True)) + finish: bool = Field(sa_column=Column(Boolean, nullable=True, default=False)) + error: str = Field(sa_column=Column(Text, nullable=True)) run_time: float = Field(default=0) @@ -52,3 +64,26 @@ class ChatInfo(BaseModel): datasource_name: str = '' datasource_exists: bool = True records: List[ChatRecord] = [] + + +class AiModelQuestion(BaseModel): + engine: str = "" + db_schema: str = "" + sql: str = "" + + def sql_sys_question(self): + return get_sql_template()['system'].format(engine=self.engine, schema=self.db_schema, question=self.question) + + def sql_user_question(self): + return get_sql_template()['user'].format(engine=self.engine, schema=self.db_schema, question=self.question) + + def chart_sys_question(self): + return get_chart_template()['system'].format(sql=self.sql, question=self.question) + + def chart_user_question(self): + return get_chart_template()['user'].format(sql=self.sql, question=self.question) + + +class ChatQuestion(AiModelQuestion): + question: str + chat_id: int diff --git a/backend/apps/chat/schemas/chat_schema.py b/backend/apps/chat/schemas/chat_schema.py deleted file mode 100644 index 57b4dee7..00000000 --- a/backend/apps/chat/schemas/chat_schema.py +++ /dev/null @@ -1,7 +0,0 @@ - -from pydantic import BaseModel - - -class ChatQuestion(BaseModel): - question: str - chat_id: int \ No newline at end of file diff --git a/backend/apps/chat/schemas/llm.py b/backend/apps/chat/schemas/llm.py deleted file mode 100644 index fb75926e..00000000 --- a/backend/apps/chat/schemas/llm.py +++ /dev/null @@ -1,197 +0,0 @@ -from langchain_community.utilities import SQLDatabase -from langgraph.prebuilt import create_react_agent -from langchain_core.prompts import ChatPromptTemplate -from apps.chat.schemas.chat_base_schema import LLMConfig, LLMFactory -from apps.chat.schemas.schema_engine import SchemaEngine -from apps.datasource.models.datasource import CoreDatasource -from apps.db.db import exec_sql, get_uri -from common.core.config import settings -import warnings -from langchain.tools import Tool -from functools import partial -import logging -from typing import AsyncGenerator -import json -import asyncio - -warnings.filterwarnings("ignore") - -class LLMService: - def __init__(self, config: LLMConfig): - # Initialize database connection - self.db = SQLDatabase.from_uri(str(settings.SQLALCHEMY_DATABASE_URI)) - - # Create LLM instance through factory - llm_instance = LLMFactory.create_llm(config) - self.llm = llm_instance.llm - - # Define prompt template - self.prompt = ChatPromptTemplate.from_messages([ - ("system", """You are a professional SQL engineer. Generate PostgreSQL SELECT queries based on the database schema and user questions. - Data modification or deletion is prohibited. Table structure is as follows: - {schema} - """), - ("human", "{question}") - ]) - - def generate_sql(self, question: str) -> str: - chain = self.prompt | self.llm - schema = self.db.get_table_info() - return chain.invoke({"schema": schema, "question": question}) - - -class AgentService: - def __init__(self, config: LLMConfig, ds: CoreDatasource): - # Initialize database connection - self.ds = ds - db_uri = get_uri(ds) - self.db = SQLDatabase.from_uri(db_uri) - # self.db = SQLDatabase.from_uri(str(settings.SQLALCHEMY_DATABASE_URI)) - - # Create LLM instance through factory - llm_instance = LLMFactory.create_llm(config) - self.llm = llm_instance.llm - - # Create a partial function of execute_sql with preset ds parameter - # bound_execute_sql = partial(execute_sql, self.ds) - bound_execute_sql = partial(execute_sql_with_db, self.db) - - # Wrap as Tool object - tools = [ - Tool( - name="execute_sql", - func=bound_execute_sql, - description="""A tool for executing SQL queries. - Input: SQL query statement (string) - Output: Query results - Example: "SELECT * FROM table_name LIMIT 5" - """ - ) - ] - - self.agent_executor = create_react_agent(self.llm, tools) - - system_prompt = """ - You are an intelligent agent capable of data analysis. When users input their data analysis requirements, - you need to first convert the requirements into executable SQL, then execute the SQL through tools to return results, - and finally summarize the SQL query results. When all tasks are completed, you need to generate an HTML format data analysis report. - - You can analyze requirements step by step to determine the final SQL query to generate. - To improve SQL generation accuracy, please evaluate the accuracy of the SQL after generation, - if there are issues, regenerate the SQL. - When SQL execution fails, you need to correct the SQL based on the error message and try to execute again. - - ### Tools ### - execute_sql: Can execute SQL by passing in SQL statements and return execution results - """ - user_prompt = """ - Below is the database information I need to query: - {schema} - - My requirement is: {question} - """ - # Define prompt template - self.prompt = ChatPromptTemplate.from_messages([ - ("system", system_prompt), - ("human", user_prompt) - ]) - - def generate_sql(self, question: str) -> str: - chain = self.prompt | self.agent_executor - schema = self.db.get_table_info() - return chain.invoke({"schema": schema, "question": question}) - - async def async_generate(self, question: str, schema: str) -> AsyncGenerator[str, None]: - - chain = self.prompt | self.agent_executor - # schema = self.db.get_table_info() - - # schema_engine = SchemaEngine(engine=self.db._engine) - # mschema = schema_engine.mschema - # mschema_str = mschema.to_mschema() - - # async for chunk in chain.astream({"schema": mschema_str, "question": question}): - async for chunk in chain.astream({"schema": schema, "question": question}): - if not isinstance(chunk, dict): - continue - - if "agent" in chunk: - messages = chunk["agent"].get("messages", []) - for msg in messages: - if tool_calls := msg.additional_kwargs.get("tool_calls"): - for tool_call in tool_calls: - response = { - "type": "tool_call", - "tool": tool_call["function"]["name"], - "args": tool_call["function"]["arguments"] - } - yield f"data: {json.dumps(response, ensure_ascii=False)}\n\n" - - if content := msg.content: - html_start = content.find("```html") - html_end = content.find("```", html_start + 6) - if html_start != -1 and html_end != -1: - html_content = content[html_start + 7:html_end].strip() - response = { - "type": "final", - "content": content.split("```html")[0].strip(), - "html": html_content - } - else: - response = { - "type": "final", - "content": content - } - yield f"data: {json.dumps(response, ensure_ascii=False)}\n\n" - - if "tools" in chunk: - messages = chunk["tools"].get("messages", []) - for msg in messages: - response = { - "type": "tool_result", - "tool": msg.name, - "content": msg.content - } - yield f"data: {json.dumps(response, ensure_ascii=False)}\n\n" - - await asyncio.sleep(0.1) - - yield f"data: {json.dumps({'type': 'complete'})}\n\n" - -def execute_sql(ds: CoreDatasource, sql: str) -> str: - """Execute SQL query - - Args: - ds: Data source instance - sql: SQL query statement - - Returns: - Query results - """ - print(f"Executing SQL on ds_id {ds.id}: {sql}") - return exec_sql(ds, sql) - -def execute_sql_with_db(db: SQLDatabase, sql: str) -> str: - """Execute SQL query using SQLDatabase - - Args: - db: SQLDatabase instance - sql: SQL query statement - - Returns: - str: Query results formatted as string - """ - try: - # Execute query - result = db.run(sql) - - if not result: - return "Query executed successfully but returned no results." - - # Format results - return str(result) - - except Exception as e: - error_msg = f"SQL execution failed: {str(e)}" - logging.error(error_msg) - raise RuntimeError(error_msg) \ No newline at end of file diff --git a/backend/apps/chat/task/__init__.py b/backend/apps/chat/task/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py new file mode 100644 index 00000000..da62eab9 --- /dev/null +++ b/backend/apps/chat/task/llm.py @@ -0,0 +1,210 @@ +import asyncio +import json +import logging +import warnings +from typing import Any, List, Union, AsyncGenerator + +from langchain_community.utilities import SQLDatabase +from langchain_core.language_models import BaseLLM +from langchain_core.messages import BaseMessage, SystemMessage, HumanMessage, AIMessage, AIMessageChunk + +from apps.ai_model.model_factory import LLMConfig, LLMFactory, get_llm_config +from apps.chat.curd.chat import save_question, save_full_sql_message, save_full_sql_message_and_answer +from apps.chat.models.chat_model import ChatQuestion, ChatRecord +from apps.datasource.models.datasource import CoreDatasource +from apps.db.db import exec_sql +from apps.system.models.system_model import AiModelDetail +from common.core.deps import SessionDep, CurrentUser + +warnings.filterwarnings("ignore") + + +class LLMService: + ds: CoreDatasource + chat_question: ChatQuestion + record: ChatRecord + config: LLMConfig + llm: BaseLLM + sql_message: List[Union[BaseMessage, dict[str, Any]]] = [] + chart_message: List[Union[BaseMessage, dict[str, Any]]] = [] + + def __init__(self, chat_question: ChatQuestion, history_records: List[ChatRecord], + ds: CoreDatasource, aimodel: AiModelDetail): + self.ds = ds + self.chat_question = chat_question + self.config = get_llm_config(aimodel) + + # Create LLM instance through factory + llm_instance = LLMFactory.create_llm(self.config) + self.llm = llm_instance.llm + + # self.agent_executor = create_react_agent(self.llm) + last_sql_messages = list( + filter(lambda r: True if r.full_sql_message is not None and r.full_sql_message.strip() != '' else False, + history_records)) + last_sql_message_str = "[]" if last_sql_messages is None or len(last_sql_messages) == 0 else last_sql_messages[ + -1].full_sql_message + last_chart_messages = list( + filter( + lambda r: True if r.full_chart_message is not None and r.full_chart_message.strip() != '' else False, + history_records)) + last_chart_message_str = "[]" if last_chart_messages is None or len(last_chart_messages) == 0 else \ + last_chart_messages[-1].full_chart_message + + last_sql_messages: List[dict[str, Any]] = json.loads(last_sql_message_str) + if last_sql_messages is None or len(last_sql_messages) == 0: + # add sys prompt + self.sql_message.append(SystemMessage(content=self.chat_question.sql_sys_question())) + else: + for last_sql_message in last_sql_messages: + _msg: BaseMessage + if last_sql_message['type'] == 'human': + _msg = HumanMessage(content=last_sql_message['content']) + elif last_sql_message['type'] == 'ai': + _msg = AIMessage(content=last_sql_message['content']) + else: + _msg = SystemMessage(content=last_sql_message['content']) + self.sql_message.append(_msg) + + last_chart_messages: List[dict[str, Any]] = json.loads(last_chart_message_str) + if last_chart_messages is None or len(last_chart_messages) == 0: + # add sys prompt + self.chart_message.append(SystemMessage(content=self.chat_question.chart_sys_question())) + else: + for last_chart_message in last_chart_messages: + _msg: BaseMessage + if last_chart_message['type'] == 'human': + _msg = HumanMessage(content=last_chart_message['content']) + elif last_chart_message['type'] == 'ai': + _msg = AIMessage(content=last_chart_message['content']) + else: + _msg = SystemMessage(content=last_chart_message['content']) + self.chart_message.append(_msg) + + def init_record(self, session: SessionDep, current_user: CurrentUser) -> ChatRecord: + self.record = save_question(session=session, current_user=current_user, question=self.chat_question) + return self.record + + def generate_sql(self, session: SessionDep): + # append current question + self.sql_message.append(HumanMessage(self.chat_question.sql_user_question())) + self.record = save_full_sql_message(session=session, record_id=self.record.id, + full_message=json.dumps( + [{'type': msg.type, 'content': msg.content} for msg in + self.sql_message], ensure_ascii=False)) + full_text = '' + res = self.llm.stream(self.sql_message) + for chunk in res: + if isinstance(chunk, dict): + full_text += chunk['content'] + yield chunk['content'] + continue + if isinstance(chunk, AIMessageChunk): + full_text += chunk.content + yield chunk.content + continue + + print(full_text) + self.sql_message.append(AIMessage(full_text)) + self.record = save_full_sql_message_and_answer(session=session, record_id=self.record.id, + answer=full_text, + full_message=json.dumps( + [{'type': msg.type, 'content': msg.content} for msg in + self.sql_message], ensure_ascii=False)) + + async def async_generate(self, question: str, schema: str) -> AsyncGenerator[str, None]: + + chain = self.prompt | self.agent_executor + # schema = self.db.get_table_info() + + # schema_engine = SchemaEngine(engine=self.db._engine) + # mschema = schema_engine.mschema + # mschema_str = mschema.to_mschema() + + # async for chunk in chain.astream({"schema": mschema_str, "question": question}): + for chunk in chain.stream({"schema": schema, "question": question}): + if not isinstance(chunk, dict): + continue + + if "agent" in chunk: + messages = chunk["agent"].get("messages", []) + for msg in messages: + if tool_calls := msg.additional_kwargs.get("tool_calls"): + for tool_call in tool_calls: + response = { + "type": "tool_call", + "tool": tool_call["function"]["name"], + "args": tool_call["function"]["arguments"] + } + yield f"data: {json.dumps(response, ensure_ascii=False)}\n\n" + + if content := msg.content: + html_start = content.find("```html") + html_end = content.find("```", html_start + 6) + if html_start != -1 and html_end != -1: + html_content = content[html_start + 7:html_end].strip() + response = { + "type": "final", + "content": content.split("```html")[0].strip(), + "html": html_content + } + else: + response = { + "type": "final", + "content": content + } + yield f"data: {json.dumps(response, ensure_ascii=False)}\n\n" + + if "tools" in chunk: + messages = chunk["tools"].get("messages", []) + for msg in messages: + response = { + "type": "tool_result", + "tool": msg.name, + "content": msg.content + } + yield f"data: {json.dumps(response, ensure_ascii=False)}\n\n" + + await asyncio.sleep(0.1) + + yield f"data: {json.dumps({'type': 'complete'})}\n\n" + + +def execute_sql(ds: CoreDatasource, sql: str) -> str: + """Execute SQL query + + Args: + ds: Data source instance + sql: SQL query statement + + Returns: + Query results + """ + print(f"Executing SQL on ds_id {ds.id}: {sql}") + return exec_sql(ds, sql) + + +def execute_sql_with_db(db: SQLDatabase, sql: str) -> str: + """Execute SQL query using SQLDatabase + + Args: + db: SQLDatabase instance + sql: SQL query statement + + Returns: + str: Query results formatted as string + """ + try: + # Execute query + result = db.run(sql) + + if not result: + return "Query executed successfully but returned no results." + + # Format results + return str(result) + + except Exception as e: + error_msg = f"SQL execution failed: {str(e)}" + logging.error(error_msg) + raise RuntimeError(error_msg) diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index db2fc95a..1e1b597f 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -220,3 +220,32 @@ def get_table_obj_by_ds(session: SessionDep, ds: CoreDatasource) -> List[TableAn fields = session.query(CoreField).filter(and_(CoreField.table_id == table.id, CoreField.checked == True)).all() _list.append(TableAndFields(schema=schema, table=table, fields=fields)) return _list + + +def get_table_schema(session: SessionDep, ds: CoreDatasource) -> str: + schema_str = "" + table_objs = get_table_obj_by_ds(session=session, ds=ds) + db_name = table_objs[0].schema + schema_str += f"【DB_ID】 {db_name}\n【Schema】\n" + for obj in table_objs: + schema_str += f"# Table: {db_name}.{obj.table.table_name}" + table_comment = '' + if obj.table.custom_comment: + table_comment = obj.table.custom_comment.strip() + if table_comment == '': + schema_str += '\n[\n' + else: + schema_str += f", {table_comment}\n[\n" + + field_list = [] + for field in obj.fields: + field_comment = '' + if field.custom_comment: + field_comment = field.custom_comment.strip() + if field_comment == '': + field_list.append(f"({field.field_name}:{field.field_type})") + else: + field_list.append(f"({field.field_name}:{field.field_type}, {field_comment})") + schema_str += ",\n".join(field_list) + schema_str += '\n]\n' + return schema_str diff --git a/backend/apps/template/__init__.py b/backend/apps/template/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/template/generate_analysis/__init__.py b/backend/apps/template/generate_analysis/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/template/generate_chart/__init__.py b/backend/apps/template/generate_chart/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/template/generate_chart/generator.py b/backend/apps/template/generate_chart/generator.py new file mode 100644 index 00000000..0962cba7 --- /dev/null +++ b/backend/apps/template/generate_chart/generator.py @@ -0,0 +1,6 @@ +from apps.template.template import get_base_template + + +def get_chart_template(): + template = get_base_template() + return template['template']['chart'] diff --git a/backend/apps/template/generate_sql/__init__.py b/backend/apps/template/generate_sql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/template/generate_sql/generator.py b/backend/apps/template/generate_sql/generator.py new file mode 100644 index 00000000..5d08cc9e --- /dev/null +++ b/backend/apps/template/generate_sql/generator.py @@ -0,0 +1,6 @@ +from apps.template.template import get_base_template + + +def get_sql_template(): + template = get_base_template() + return template['template']['sql'] diff --git a/backend/apps/template/template.py b/backend/apps/template/template.py new file mode 100644 index 00000000..7dd5b1cc --- /dev/null +++ b/backend/apps/template/template.py @@ -0,0 +1,15 @@ +import yaml + +base_template = None + + +def load(): + with open('./template.yaml', 'r') as f: + global base_template + base_template = yaml.load(f, Loader=yaml.SafeLoader) + + +def get_base_template(): + if not base_template: + load() + return base_template diff --git a/backend/pyproject.toml b/backend/pyproject.toml index f1f911ac..9d4acbc1 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -32,7 +32,8 @@ dependencies = [ "pandas (>=2.2.3,<3.0.0)", "openpyxl (>=3.1.5,<4.0.0)", "psycopg2-binary (>=2.9.10,<3.0.0)", - "oracledb (>=3.1.1,<4.0.0)" + "oracledb (>=3.1.1,<4.0.0)", + "pyyaml (>=6.0.2,<7.0.0)" ] [[tool.uv.index]] url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" diff --git a/backend/template.yaml b/backend/template.yaml new file mode 100644 index 00000000..60dee447 --- /dev/null +++ b/backend/template.yaml @@ -0,0 +1,52 @@ +template: + sql: + system: | + 任务: + 根据表结构和问题生成符合{engine}数据库引擎规范的sql语句。 + 你必须遵守以下规则: + - 生成的SQL必须符合{engine}的规范。 + - 根据表结构生成SQL语句。 + - SQL查询中不能使用星号(*)，必须明确指定字段名. + - SQL查询的字段名和别名必须为英文。 + - 计算占比，百分比类型字段，保留两位小数，以%结尾。 + - 问题与生成SQL无关时，直接回答：抱歉，我无法回答您的问题。 + user: | + 表结构: + {schema} + + 问题: + {question} + chart: + system: | + ### 说明： + 您的任务是通过给定的问题和SQL生成 JSON 以进行数据可视化。 + 请遵守以下规则： + - 如果需要表格，则生成的 JSON 格式应为： + {{"type":"table", "title": "标题", "columns": [{{"name":"中文字段名1", "value": "SQL 查询列 1(有别名用别名)"}}, {{"name": "中文字段名 2", "value": "SQL 查询列 2(有别名用别名)"}}]}} + 必须从 SQL 查询列中提取“columns”。 + - 如果需要条形图，则生成的 JSON 格式应为： + {{"type":"bar", "title": "标题", "axis": {{"x": {{"name":"x轴的中文名称", "value": "SQL 查询 x 轴的列(有别名用别名)"}}, "y": {{"name":"y轴的中文名称","value": "SQL 查询 y 轴的列(有别名用别名)"}}}} + 必须从 SQL 查询列中提取“x”和“y”。 + - 如果需要折线图，则生成的 JSON 格式应为： + {{"type":"line", "title": "标题", "axis": {{"x": {{"name":"x轴的中文名称","value": "x轴的SQL查询列(有别名用别名)"}}, "y": {{"name":"y轴的中文名称","value": "y轴的SQL查询列(有别名用别名)"}}}} + 其中“x”和“y”必须从SQL查询列中提取。 + - 如果需要饼图，则生成的 JSON 格式应为： + {{"type":"pie", "title": "标题", "column": {{"name":"中文字段名","value":"SQL查询列1(有别名用别名)"}}}} + 其中“column”必须从SQL查询列中提取。 + - 如果答案未知或者与生成JSON无关，则生成的 JSON 格式应为： + {{"type":"error", "reason": "抱歉，我无法回答您的问题。"}} + + ### 示例 + 如果 SQL 为： SELECT products_sales_data.category, AVG(products_sales_data.price) AS average_price FROM products_sales_data GROUP BY products_sales_data.category; + 问题是：每个商品分类的平均价格 + 则生成的 JSON 可以是： {{"type":"table", "title": "每个商品分类的平均价格", "columns": [{{"name":"商品分类","value":"category"}}, {{"name":"平均价格","value":"average_price"}}]}} + + ### 响应： + 根据您的指示，这是我生成的与 问题 和 sql 匹配的 JSON： + ```json + user: | + ### SQL: + {sql} + + ### 问题： + {question} \ No newline at end of file diff --git a/frontend/components.d.ts b/frontend/components.d.ts deleted file mode 100644 index ccfedf7f..00000000 --- a/frontend/components.d.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* eslint-disable */ -/* prettier-ignore */ -// @ts-nocheck -// Generated by unplugin-vue-components -// Read more: https://github.com/vuejs/core/pull/3399 -import '@vue/runtime-core' - -export {} - -declare module '@vue/runtime-core' { - export interface GlobalComponents { - ElAside: typeof import('element-plus-secondary/es')['ElAside'] - ElAvatar: typeof import('element-plus-secondary/es')['ElAvatar'] - ElButton: typeof import('element-plus-secondary/es')['ElButton'] - ElCheckbox: typeof import('element-plus-secondary/es')['ElCheckbox'] - ElContainer: typeof import('element-plus-secondary/es')['ElContainer'] - ElDialog: typeof import('element-plus-secondary/es')['ElDialog'] - ElDivider: typeof import('element-plus-secondary/es')['ElDivider'] - ElDropdown: typeof import('element-plus-secondary/es')['ElDropdown'] - ElDropdownItem: typeof import('element-plus-secondary/es')['ElDropdownItem'] - ElDropdownMenu: typeof import('element-plus-secondary/es')['ElDropdownMenu'] - ElFooter: typeof import('element-plus-secondary/es')['ElFooter'] - ElForm: typeof import('element-plus-secondary/es')['ElForm'] - ElFormItem: typeof import('element-plus-secondary/es')['ElFormItem'] - ElHeader: typeof import('element-plus-secondary/es')['ElHeader'] - ElIcon: typeof import('element-plus-secondary/es')['ElIcon'] - ElInput: typeof import('element-plus-secondary/es')['ElInput'] - ElMain: typeof import('element-plus-secondary/es')['ElMain'] - ElMenu: typeof import('element-plus-secondary/es')['ElMenu'] - ElMenuItem: typeof import('element-plus-secondary/es')['ElMenuItem'] - ElOption: typeof import('element-plus-secondary/es')['ElOption'] - ElPagination: typeof import('element-plus-secondary/es')['ElPagination'] - ElScrollbar: typeof import('element-plus-secondary/es')['ElScrollbar'] - ElSelect: typeof import('element-plus-secondary/es')['ElSelect'] - ElSlider: typeof import('element-plus-secondary/es')['ElSlider'] - ElSwitch: typeof import('element-plus-secondary/es')['ElSwitch'] - ElTable: typeof import('element-plus-secondary/es')['ElTable'] - ElTableColumn: typeof import('element-plus-secondary/es')['ElTableColumn'] - ElTabPane: typeof import('element-plus-secondary/es')['ElTabPane'] - ElTabs: typeof import('element-plus-secondary/es')['ElTabs'] - ElTooltip: typeof import('element-plus-secondary/es')['ElTooltip'] - Icon: typeof import('./src/components/icon-custom/src/Icon.vue')['default'] - Layout: typeof import('./src/components/layout/index.vue')['default'] - RouterLink: typeof import('vue-router')['RouterLink'] - RouterView: typeof import('vue-router')['RouterView'] - SinglePage: typeof import('./src/components/layout/SinglePage.vue')['default'] - } - export interface ComponentCustomProperties { - vLoading: typeof import('element-plus-secondary/es')['ElLoadingDirective'] - } -} diff --git a/frontend/src/views/system/model/index.vue b/frontend/src/views/system/model/index.vue index 72789b8e..02982c2a 100644 --- a/frontend/src/views/system/model/index.vue +++ b/frontend/src/views/system/model/index.vue @@ -119,13 +119,13 @@ - +