klinsmaya/SQLBot
1
0
Fork 0
mirror of https://gh-proxy.org/https://github.com/klinsmaya/SQLBot.git synced 2026-11-08 01:06:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Log file

Browse Source
This commit is contained in:
fit2cloud-chenyw
2025-07-23 15:59:23 +08:00
parent 9beb4f8089
commit 63a1d1fe7d
12 changed files with 200 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+6
View File
@@ -14,6 +14,12 @@ FIRST_SUPERUSER_PASSWORD=123456 # Change this to your pwd
TOKEN_KEY="X-SQLBOT-TOKEN"
DEFAULT_PWD="SQLBot@123456"
ASSISTANT_TOKEN_KEY="X-SQLBOT-ASSISTANT-TOKEN"
LOG_LEVEL="INFO" # DEBUG, INFO, WARNING, ERROR
LOG_DIR="/opt/sqlbot/logs"
LOG_FORMAT="%(asctime)s - %(name)s - %(levelname)s:%(lineno)d - %(message)s"
SQL_DEBUG=False
# Postgres
POSTGRES_SERVER=localhost
backend/alembic/env.py
-2
View File
@@ -5,7 +5,6 @@ from os.path import abspath, dirname
sys.path.insert(0, dirname(dirname(abspath(__file__))))
import os
from logging.config import fileConfig
from alembic import context
from sqlalchemy import engine_from_config, pool
@@ -16,7 +15,6 @@ config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
backend/apps/chat/task/llm.py
+18 -19
View File
@@ -1,6 +1,5 @@
import json
import sqlparse
import logging
import traceback
import warnings
from typing import Any, List, Optional, Union, Dict
@@ -35,7 +34,7 @@ from apps.system.crud.assistant import AssistantOutDs, AssistantOutDsFactory, ge
from apps.system.schemas.system_schema import AssistantOutDsSchema
from common.core.config import settings
from common.core.deps import CurrentAssistant, SessionDep, CurrentUser
from common.utils.utils import extract_nested_json
from common.utils.utils import SQLBotLogUtil, extract_nested_json
warnings.filterwarnings("ignore")
@@ -223,7 +222,7 @@ class LLMService:
res = self.llm.stream(analysis_msg)
token_usage = {}
for chunk in res:
print(chunk)
SQLBotLogUtil.info(chunk)
reasoning_content_chunk = ''
if 'reasoning_content' in chunk.additional_kwargs:
reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '')
@@ -273,7 +272,7 @@ class LLMService:
res = self.llm.stream(predict_msg)
token_usage = {}
for chunk in res:
print(chunk)
SQLBotLogUtil.info(chunk)
reasoning_content_chunk = ''
if 'reasoning_content' in chunk.additional_kwargs:
reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '')
@@ -323,7 +322,7 @@ class LLMService:
token_usage = {}
res = self.llm.stream(guess_msg)
for chunk in res:
print(chunk)
SQLBotLogUtil.info(chunk)
reasoning_content_chunk = ''
if 'reasoning_content' in chunk.additional_kwargs:
reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '')
@@ -390,7 +389,7 @@ class LLMService:
token_usage = {}
res = self.llm.stream(datasource_msg)
for chunk in res:
print(chunk)
SQLBotLogUtil.info(chunk)
reasoning_content_chunk = ''
if 'reasoning_content' in chunk.additional_kwargs:
reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '')
@@ -474,7 +473,7 @@ class LLMService:
token_usage = {}
res = self.llm.stream(self.sql_message)
for chunk in res:
print(chunk)
SQLBotLogUtil.info(chunk)
reasoning_content_chunk = ''
if 'reasoning_content' in chunk.additional_kwargs:
reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '')
@@ -543,7 +542,7 @@ class LLMService:
res = self.llm.stream(msg)
token_usage = {}
for chunk in res:
print(chunk)
SQLBotLogUtil.info(chunk)
reasoning_content_chunk = ''
if 'reasoning_content' in chunk.additional_kwargs:
reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '')
@@ -567,7 +566,7 @@ class LLMService:
# 'content': msg.content} for msg
# in
# analysis_msg]).decode())
print(full_filter_text)
SQLBotLogUtil.info(full_filter_text)
return full_filter_text
def generate_chart(self):
@@ -582,7 +581,7 @@ class LLMService:
token_usage = {}
res = self.llm.stream(self.chart_message)
for chunk in res:
print(chunk)
SQLBotLogUtil.info(chunk)
reasoning_content_chunk = ''
if 'reasoning_content' in chunk.additional_kwargs:
reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '')
@@ -691,7 +690,7 @@ class LLMService:
Returns:
Query results
"""
print(f"Executing SQL on ds_id {self.ds.id}: {sql}")
SQLBotLogUtil.info(f"Executing SQL on ds_id {self.ds.id}: {sql}")
return exec_sql(self.ds, sql)
@@ -717,7 +716,7 @@ def execute_sql_with_db(db: SQLDatabase, sql: str) -> str:
except Exception as e:
error_msg = f"SQL execution failed: {str(e)}"
logging.error(error_msg)
SQLBotLogUtil.exception(error_msg)
raise RuntimeError(error_msg)
@@ -741,7 +740,7 @@ def run_task(llm_service: LLMService, in_chat: bool = True):
ds_res = llm_service.select_datasource()
for chunk in ds_res:
print(chunk)
SQLBotLogUtil.info(chunk)
if in_chat:
yield orjson.dumps(
{'content': chunk.get('content'), 'reasoning_content': chunk.get('reasoning_content'),
@@ -765,7 +764,7 @@ def run_task(llm_service: LLMService, in_chat: bool = True):
yield orjson.dumps({'type': 'info', 'msg': 'sql generated'}).decode() + '\n\n'
# filter sql
print(full_sql_text)
SQLBotLogUtil.info(full_sql_text)
# todo row permission
sql_json_str = extract_nested_json(full_sql_text)
@@ -785,11 +784,11 @@ def run_task(llm_service: LLMService, in_chat: bool = True):
raise Exception("SQL query is empty")
sql_result = llm_service.generate_filter(data.get('sql'), data.get('tables')) # maybe no sql and tables
print(sql_result)
SQLBotLogUtil.info(sql_result)
sql = llm_service.check_save_sql(res=sql_result)
# sql = llm_service.check_save_sql(res=full_sql_text)
print(sql)
SQLBotLogUtil.info(sql)
format_sql = sqlparse.format(sql, reindent=True)
if in_chat:
yield orjson.dumps({'content': format_sql, 'type': 'sql'}).decode() + '\n\n'
@@ -815,9 +814,9 @@ def run_task(llm_service: LLMService, in_chat: bool = True):
yield orjson.dumps({'type': 'info', 'msg': 'chart generated'}).decode() + '\n\n'
# filter chart
print(full_chart_text)
SQLBotLogUtil.info(full_chart_text)
chart = llm_service.check_save_chart(res=full_chart_text)
print(chart)
SQLBotLogUtil.info(chart)
if in_chat:
yield orjson.dumps({'content': orjson.dumps(chart).decode(), 'type': 'chart'}).decode() + '\n\n'
else:
@@ -856,7 +855,7 @@ def run_task(llm_service: LLMService, in_chat: bool = True):
if chart['type'] != 'table':
yield '### generated chart picture\n\n'
image_url = request_picture(llm_service.record.chat_id, llm_service.record.id, chart, result)
print(image_url)
SQLBotLogUtil.info(image_url)
yield f'![{chart["type"]}]({image_url})'
except Exception as e:
traceback.print_exc()
backend/apps/datasource/crud/datasource.py
+3 -3
View File
@@ -15,7 +15,7 @@ from apps.db.db import get_engine, get_tables, get_fields, exec_sql
from apps.db.engine import get_engine_config, get_engine_conn
from apps.db.type import db_type_relation
from common.core.deps import SessionDep, CurrentUser, Trans
from common.utils.utils import deepcopy_ignore_extra
from common.utils.utils import SQLBotLogUtil, deepcopy_ignore_extra
from .table import get_tables_by_ds_id
from ..crud.field import delete_field_by_ds_id, update_field
from ..crud.table import delete_table_by_ds_id, update_table
@@ -39,12 +39,12 @@ def check_status(session: SessionDep, ds: CoreDatasource, is_raise: bool = False
conn = get_engine(ds)
try:
with conn.connect() as connection:
print("success")
SQLBotLogUtil.info("success")
return True
except Exception as e:
SQLBotLogUtil.error(f"Datasource {ds.id} connection failed: {e}")
if is_raise:
raise HTTPException(status_code=500, detail=f'Connect Failed: {e.args}')
print("Fail:", e)
return False
backend/apps/system/crud/user.py
+2 -2
View File
@@ -1,13 +1,13 @@
from typing import Optional
from sqlmodel import Session, select, delete as sqlmodel_delete
import logging
from apps.system.models.system_model import UserWsModel, WorkspaceModel
from apps.system.schemas.auth import CacheName, CacheNamespace
from apps.system.schemas.system_schema import BaseUserDTO, UserInfoDTO, UserWs
from common.core.deps import SessionDep
from common.core.sqlbot_cache import cache, clear_cache
from common.utils.locale import I18n
from common.utils.utils import SQLBotLogUtil
from ..models.user import UserModel
from common.core.security import verify_md5pwd
@@ -68,4 +68,4 @@ async def single_delete(session: SessionDep, id: int):
#@clear_cache(namespace=CacheNamespace.AUTH_INFO, cacheName=CacheName.USER_INFO, keyExpression="id")
async def clean_user_cache(id: int):
logging.info(f"User cache for [{id}] has been cleaned")
SQLBotLogUtil.info(f"User cache for [{id}] has been cleaned")
backend/apps/system/middleware/auth.py
+5 -1
View File
@@ -1,6 +1,6 @@
from typing import Optional
from fastapi import HTTPException, Request
from fastapi import Request
from fastapi.responses import JSONResponse
import jwt
from sqlmodel import Session
@@ -13,6 +13,7 @@ from apps.system.schemas.system_schema import AssistantHeader, UserInfoDTO
from common.core import security
from common.core.config import settings
from common.core.schemas import TokenPayload
from common.utils.utils import SQLBotLogUtil
from common.utils.whitelist import whiteUtils
from fastapi.security.utils import get_authorization_scheme_param
class TokenMiddleware(BaseHTTPMiddleware):
@@ -70,6 +71,7 @@ class TokenMiddleware(BaseHTTPMiddleware):
) """
return True, session_user
except Exception as e:
SQLBotLogUtil.exception(f"Token validation error: {str(e)}", exc_info=True)
return False, e
@@ -95,4 +97,6 @@ class TokenMiddleware(BaseHTTPMiddleware):
assistant_info = AssistantHeader.model_validate(assistant_info.model_dump(exclude_unset=True))
return True, session_user, assistant_info
except Exception as e:
SQLBotLogUtil.exception(f"Assistant validation error: {str(e)}", exc_info=True)
# Return False and the exception message
return False, e
backend/common/core/config.py
+5
View File
@@ -60,6 +60,11 @@ class Settings(BaseSettings):
TOKEN_KEY: str = "X-SQLBOT-TOKEN"
DEFAULT_PWD: str = "SQLBot@123456"
ASSISTANT_TOKEN_KEY: str = "X-SQLBOT-ASSISTANT-TOKEN"
LOG_LEVEL: str = "INFO" # DEBUG, INFO, WARNING, ERROR
LOG_DIR: str = "logs"
LOG_FORMAT: str = "%(asctime)s - %(name)s - %(levelname)s:%(lineno)d - %(message)s"
SQL_DEBUG: bool = False
@computed_field # type: ignore[prop-decorator]
@property
backend/common/core/response_middleware.py
+4 -4
View File
@@ -1,10 +1,10 @@
import json
import logging
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.responses import JSONResponse
from starlette.exceptions import HTTPException
from starlette.requests import Request
from common.core.config import settings
from common.utils.utils import SQLBotLogUtil
class ResponseMiddleware(BaseHTTPMiddleware):
def __init__(self, app):
super().__init__(app)
@@ -48,7 +48,7 @@ class ResponseMiddleware(BaseHTTPMiddleware):
}
)
except Exception as e:
logging.error(f"Response processing error: {str(e)}", exc_info=True)
SQLBotLogUtil.error(f"Response processing error: {str(e)}", exc_info=True)
return JSONResponse(
status_code=500,
content={
@@ -68,6 +68,7 @@ class ResponseMiddleware(BaseHTTPMiddleware):
class exception_handler():
@staticmethod
async def http_exception_handler(request: Request, exc: HTTPException):
SQLBotLogUtil.exception(f"HTTP Exception: {exc.detail}", exc_info=True)
return JSONResponse(
status_code=exc.status_code,
content={
@@ -81,8 +82,7 @@ class exception_handler():
@staticmethod
async def global_exception_handler(request: Request, exc: Exception):
logging.error(f"Error info: {str(exc)}", exc_info=True)
SQLBotLogUtil.exception(f"Unhandled Exception: {str(exc)}", exc_info=True)
return JSONResponse(
status_code=500,
content={
backend/common/core/sqlbot_cache.py
+8 -11
View File
@@ -1,12 +1,13 @@
import logging
from fastapi_cache import FastAPICache
from fastapi_cache.decorator import cache as original_cache
from functools import partial, wraps
from typing import Optional, Set, Any, Dict, Tuple, Callable
from typing import Optional, Set, Any, Dict, Tuple
from inspect import Parameter, signature
from contextlib import asynccontextmanager
import asyncio
from common.utils.utils import SQLBotLogUtil
# 锁管理
_cache_locks = {}
@@ -65,7 +66,7 @@ def custom_key_builder(
base_key += f"{value}:"
except (IndexError, KeyError, AttributeError) as e:
logging.warning(f"Failed to evaluate keyExpression '{keyExpression}': {str(e)}")
SQLBotLogUtil.warning(f"Failed to evaluate keyExpression '{keyExpression}': {str(e)}")
return base_key
@@ -127,19 +128,16 @@ def cache(
)
async with _get_cache_lock(cache_key):
logging.info(f"Using cache key: {cache_key}")
print(f"Using cache key: {cache_key}")
SQLBotLogUtil.info(f"Using cache key: {cache_key}")
backend = FastAPICache.get_backend()
cached_value = await backend.get(cache_key)
if cached_value is not None:
logging.info(f"Cache hit for key: {cache_key}, the value is: {cached_value}")
print(f"Cache hit for key: {cache_key}, the value is: {cached_value}")
SQLBotLogUtil.info(f"Cache hit for key: {cache_key}, the value is: {cached_value}")
return cached_value
result = await func(*args, **kwargs)
await backend.set(cache_key, result, expire)
logging.info(f"Cache miss for key: {cache_key}, result cached.")
print(f"Cache miss for key: {cache_key}, result cached.")
SQLBotLogUtil.info(f"Cache miss for key: {cache_key}, result cached.")
return result
return wrapper
@@ -167,8 +165,7 @@ def clear_cache(
if await FastAPICache.get_backend().get(cache_key):
await FastAPICache.clear(key=cache_key)
result = await func(*args, **kwargs)
logging.info(f"Clearing cache for key: {cache_key}")
print(f"Clearing cache for key: {cache_key}")
SQLBotLogUtil.info(f"Clearing cache for key: {cache_key}")
return result
return await func(*args, **kwargs)
backend/common/utils/utils.py
+143 -3
View File
@@ -1,6 +1,10 @@
import hashlib
import inspect
import logging
from datetime import datetime, timedelta, timezone
from logging.handlers import RotatingFileHandler
from pathlib import Path
from common.core.config import settings
from typing import Optional
import jwt
@@ -10,9 +14,6 @@ from jwt.exceptions import InvalidTokenError
from common.core import security
from common.core.config import settings
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
def generate_password_reset_token(email: str) -> str:
delta = timedelta(hours=settings.EMAIL_RESET_TOKEN_EXPIRE_HOURS)
@@ -76,3 +77,142 @@ def extract_nested_json(text):
def string_to_numeric_hash(text: str, bits: Optional[int] = 64) -> int:
hash_bytes = hashlib.sha256(text.encode()).digest()
return int.from_bytes(hash_bytes[:bits//8], byteorder='big')
def setup_logging():
# 确保日志目录存在
log_dir = Path(settings.LOG_DIR)
log_dir.mkdir(parents=True, exist_ok=True)
# 日志格式
formatter = logging.Formatter(
f'{settings.LOG_FORMAT}'
)
# 控制台日志
console_handler = logging.StreamHandler()
console_handler.setLevel(settings.LOG_LEVEL)
console_handler.setFormatter(formatter)
# 文件日志处理器
file_handlers = {
'debug': logging.DEBUG,
'info': logging.INFO,
'warn': logging.WARNING,
'error': logging.ERROR
}
# 主日志记录器
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG) # 设置最低级别
# 添加控制台处理器
root_logger.addHandler(console_handler)
# 为每个级别创建文件处理器
for level_name, level in file_handlers.items():
file_path = log_dir / f"{level_name}.log"
handler = RotatingFileHandler(
file_path,
maxBytes=10 * 1024 * 1024, # 10MB
backupCount=5,
encoding='utf-8'
)
handler.setLevel(level)
handler.setFormatter(formatter)
# 添加过滤器只处理特定级别日志
if level_name == 'debug':
handler.addFilter(lambda record: record.levelno == logging.DEBUG)
elif level_name == 'info':
handler.addFilter(lambda record: record.levelno == logging.INFO)
elif level_name == 'warn':
handler.addFilter(lambda record: record.levelno == logging.WARNING)
elif level_name == 'error':
handler.addFilter(lambda record: record.levelno >= logging.ERROR)
root_logger.addHandler(handler)
# SQL日志特殊处理
if settings.LOG_LEVEL == "DEBUG" and settings.SQL_DEBUG:
sql_logger = logging.getLogger('sqlalchemy.engine')
sql_logger.setLevel(logging.DEBUG)
sql_handler = RotatingFileHandler(
log_dir / "sql.log",
maxBytes=10 * 1024 * 1024,
backupCount=2,
encoding='utf-8'
)
sql_handler.setFormatter(formatter)
sql_logger.addHandler(sql_handler)
setup_logging()
class CallerLogger(logging.Logger):
def __init__(self, logger: logging.Logger):
self.logger = logger
super().__init__(logger.name, logger.level)
def _log(self, level, msg, args, exc_info=None, extra=None, stacklevel=3):
if self.logger.isEnabledFor(level):
self.logger._log(level, msg, args, exc_info=exc_info, extra=extra, stacklevel=stacklevel)
class SQLBotLogUtil:
@staticmethod
def _get_logger() -> logging.Logger:
frame = inspect.currentframe()
try:
caller_frame = frame.f_back.f_back
module_name = caller_frame.f_globals.get('__name__', '__main__')
return CallerLogger(logging.getLogger(module_name))
finally:
del frame
@staticmethod
def debug(msg: str, *args, **kwargs):
logger = SQLBotLogUtil._get_logger()
if logger.isEnabledFor(logging.DEBUG):
logger._log(logging.DEBUG, msg, args, **kwargs)
@staticmethod
def info(msg: str, *args, **kwargs):
logger = SQLBotLogUtil._get_logger()
if logger.isEnabledFor(logging.INFO):
logger._log(logging.INFO, msg, args, **kwargs)
@staticmethod
def warning(msg: str, *args, **kwargs):
logger = SQLBotLogUtil._get_logger()
if logger.isEnabledFor(logging.WARNING):
logger._log(logging.WARNING, msg, args, **kwargs)
@staticmethod
def error(msg: str, *args, exc_info: Optional[bool] = None, **kwargs):
logger = SQLBotLogUtil._get_logger()
if logger.isEnabledFor(logging.ERROR):
logger._log(
logging.ERROR,
msg,
args,
exc_info=exc_info if exc_info is not None else True,
**kwargs
)
@staticmethod
def exception(msg: str, *args, **kwargs):
logger = SQLBotLogUtil._get_logger()
if logger.isEnabledFor(logging.ERROR):
logger._log(logging.ERROR, msg, args, exc_info=True, **kwargs)
@staticmethod
def critical(msg: str, *args, **kwargs):
logger = SQLBotLogUtil._get_logger()
if logger.isEnabledFor(logging.CRITICAL):
logger._log(logging.CRITICAL, msg, args, **kwargs)
backend/common/utils/whitelist.py
+2 -1
View File
@@ -2,6 +2,7 @@
import re
from typing import List, Pattern
from common.core.config import settings
from common.utils.utils import SQLBotLogUtil
wlist = [
"/",
"/docs",
@@ -50,7 +51,7 @@ class WhitelistChecker:
try:
self._compiled_patterns.append(re.compile(regex_pattern))
except re.error:
print(f"Invalid regular expression pattern: {pattern}")
SQLBotLogUtil.error(f"Invalid regex pattern: {regex_pattern}")
def is_whitelisted(self, path: str) -> bool:
prefix = settings.API_V1_STR
backend/main.py
+4 -3
View File
@@ -1,4 +1,3 @@
import logging
from fastapi.concurrency import asynccontextmanager
import sentry_sdk
from fastapi import FastAPI
@@ -18,6 +17,8 @@ import sqlbot_xpack
from fastapi_cache import FastAPICache
from fastapi_cache.backends.inmemory import InMemoryBackend
from common.utils.utils import SQLBotLogUtil
def run_migrations():
alembic_cfg = Config("alembic.ini")
command.upgrade(alembic_cfg, "head")
@@ -27,10 +28,10 @@ def run_migrations():
async def lifespan(app: FastAPI):
run_migrations()
FastAPICache.init(InMemoryBackend())
print("✅ FastAPICache 初始化完成")
init_dynamic_cors(app)
SQLBotLogUtil.info("✅ SQLBot 初始化完成")
yield
SQLBotLogUtil.info("SQLBot 应用关闭")
def custom_generate_unique_id(route: APIRoute) -> str:
tag = route.tags[0] if route.tags and len(route.tags) > 0 else ""