klinsmaya/SQLBot
1
0
Fork 0
mirror of https://gh-proxy.org/https://github.com/klinsmaya/SQLBot.git synced 2026-11-09 09:46:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: import Terminologies

Browse Source
This commit is contained in:
ulleo
2025-11-18 17:00:55 +08:00
committed by ulleo
parent d1440ca1c3
commit e321abde0b
8 changed files with 470 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/apps/terminology/api/terminology.py
+116 -2
View File
@@ -1,15 +1,20 @@
import asyncio
import hashlib
import io
import os
import uuid
from http.client import HTTPException
from typing import Optional
import pandas as pd
from fastapi import APIRouter, Query
from fastapi import APIRouter, File, UploadFile, Query
from fastapi.responses import StreamingResponse
from apps.chat.models.chat_model import AxisObj
from apps.terminology.curd.terminology import page_terminology, create_terminology, update_terminology, \
delete_terminology, enable_terminology, get_all_terminology
delete_terminology, enable_terminology, get_all_terminology, batch_create_terminology
from apps.terminology.models.terminology_model import TerminologyInfo
from common.core.config import settings
from common.core.deps import SessionDep, CurrentUser, Trans
from common.utils.data_format import DataFormat
@@ -89,3 +94,112 @@ async def export_excel(session: SessionDep, trans: Trans, current_user: CurrentU
result = await asyncio.to_thread(inner)
return StreamingResponse(result, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
path = settings.EXCEL_PATH
from sqlalchemy.orm import sessionmaker, scoped_session
from common.core.db import engine
from sqlmodel import Session
session_maker = scoped_session(sessionmaker(bind=engine, class_=Session))
@router.post("/uploadExcel")
async def upload_excel(trans: Trans, current_user: CurrentUser, file: UploadFile = File(...)):
ALLOWED_EXTENSIONS = {"xlsx", "xls"}
if not file.filename.lower().endswith(tuple(ALLOWED_EXTENSIONS)):
raise HTTPException(400, "Only support .xlsx/.xls")
os.makedirs(path, exist_ok=True)
base_filename = f"{file.filename.split('.')[0]}_{hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}"
filename = f"{base_filename}.{file.filename.split('.')[1]}"
save_path = os.path.join(path, filename)
with open(save_path, "wb") as f:
f.write(await file.read())
oid = current_user.oid
use_cols = [0, 1, 2, 3, 4]
def inner():
session = session_maker()
sheet_names = pd.ExcelFile(save_path).sheet_names
import_data = []
for sheet_name in sheet_names:
df = pd.read_excel(
save_path,
sheet_name=sheet_name,
engine='calamine',
header=0,
usecols=use_cols,
dtype=str
).fillna("")
for index, row in df.iterrows():
# 跳过空行
if row.isnull().all():
continue
word = row[0].strip() if pd.notna(row[0]) and row[0].strip() else None
other_words = [w.strip() for w in row[1].strip().split(',')] if pd.notna(row[1]) and row[
1].strip() else []
description = row[2].strip() if pd.notna(row[2]) and row[2].strip() else None
datasource_names = [d.strip() for d in row[3].strip().split(',')] if pd.notna(row[3]) and row[
3].strip() else []
all_datasource = True if pd.notna(row[4]) and row[4].lower().strip() in ['y', 'yes', 'true'] else False
specific_ds = False if all_datasource else True
import_data.append(TerminologyInfo(word=word, description=description, other_words=other_words,
datasource_names=datasource_names, specific_ds=specific_ds))
res = batch_create_terminology(session, import_data, oid, trans)
failed_records = res['failed_records']
error_excel_filename = None
if len(failed_records) > 0:
data_list = []
for obj in failed_records:
_data = {
"word": obj['data'].word,
"other_words": ', '.join(obj['data'].other_words) if obj['data'].other_words else '',
"description": obj['data'].description,
"all_data_sources": 'N' if obj['data'].specific_ds else 'Y',
"datasource": ', '.join(obj['data'].datasource_names) if obj['data'].datasource_names and obj[
'data'].specific_ds else '',
"errors": obj['errors']
}
data_list.append(_data)
fields = []
fields.append(AxisObj(name=trans('i18n_terminology.term_name'), value='word'))
fields.append(AxisObj(name=trans('i18n_terminology.synonyms'), value='other_words'))
fields.append(AxisObj(name=trans('i18n_terminology.term_description'), value='description'))
fields.append(AxisObj(name=trans('i18n_terminology.effective_data_sources'), value='datasource'))
fields.append(AxisObj(name=trans('i18n_terminology.all_data_sources'), value='all_data_sources'))
fields.append(AxisObj(name=trans('i18n_data_training.error_info'), value='errors'))
md_data, _fields_list = DataFormat.convert_object_array_for_pandas(fields, data_list)
df = pd.DataFrame(md_data, columns=_fields_list)
error_excel_filename = f"{base_filename}_error.xlsx"
save_error_path = os.path.join(path, error_excel_filename)
# 保存 DataFrame 到 Excel
df.to_excel(save_error_path, index=False)
return {
'success_count': res['success_count'],
'failed_count': len(failed_records),
'duplicate_count': res['duplicate_count'],
'original_count': res['original_count'],
'error_excel_filename': error_excel_filename,
}
return await asyncio.to_thread(inner)
backend/apps/terminology/curd/terminology.py
+216 -21
View File
@@ -201,6 +201,16 @@ def get_all_terminology(session: SessionDep, name: Optional[str] = None, oid: Op
def create_terminology(session: SessionDep, info: TerminologyInfo, oid: int, trans: Trans):
"""
创建单个术语记录
"""
# 基本验证
if not info.word or not info.word.strip():
raise Exception(trans("i18n_terminology.word_cannot_be_empty"))
if not info.description or not info.description.strip():
raise Exception(trans("i18n_terminology.description_cannot_be_empty"))
create_time = datetime.datetime.now()
specific_ds = info.specific_ds if info.specific_ds is not None else False
@@ -210,16 +220,26 @@ def create_terminology(session: SessionDep, info: TerminologyInfo, oid: int, tra
if not datasource_ids:
raise Exception(trans("i18n_terminology.datasource_cannot_be_none"))
parent = Terminology(word=info.word, create_time=create_time, description=info.description, oid=oid,
specific_ds=specific_ds, enabled=info.enabled,
datasource_ids=datasource_ids)
parent = Terminology(
word=info.word,
create_time=create_time,
description=info.description,
oid=oid,
specific_ds=specific_ds,
enabled=info.enabled,
datasource_ids=datasource_ids
)
words = [info.word]
for child in info.other_words:
if child in words:
for child_word in info.other_words:
# 先检查是否为空字符串
if not child_word or child_word.strip() == "":
continue
if child_word in words:
raise Exception(trans("i18n_terminology.cannot_be_repeated"))
else:
words.append(child)
words.append(child_word)
# 基础查询条件（word 和 oid 必须满足）
base_query = and_(
@@ -243,7 +263,7 @@ def create_terminology(session: SessionDep, info: TerminologyInfo, oid: int, tra
SELECT 1 FROM jsonb_array_elements(datasource_ids) AS elem
WHERE elem::text::int = ANY(:datasource_ids)
)
""") # 检查是否包含任意目标值
""")
)
)
)
@@ -255,31 +275,206 @@ def create_terminology(session: SessionDep, info: TerminologyInfo, oid: int, tra
if exists:
raise Exception(trans("i18n_terminology.exists_in_db"))
result = Terminology(**parent.model_dump())
session.add(parent)
session.flush()
session.refresh(parent)
result.id = parent.id
session.commit()
_list: List[Terminology] = []
# 插入子记录（其他词）
child_list = []
if info.other_words:
for other_word in info.other_words:
if other_word.strip() == "":
continue
_list.append(
Terminology(pid=result.id, word=other_word, create_time=create_time, oid=oid, enabled=result.enabled,
specific_ds=specific_ds, datasource_ids=datasource_ids))
session.bulk_save_objects(_list)
session.flush()
child_list.append(
Terminology(
pid=parent.id,
word=other_word,
create_time=create_time,
oid=oid,
enabled=info.enabled,
specific_ds=specific_ds,
datasource_ids=datasource_ids
)
)
if child_list:
session.bulk_save_objects(child_list)
session.flush()
session.commit()
# embedding
run_save_terminology_embeddings([result.id])
# 处理embedding
run_save_terminology_embeddings([parent.id])
return result.id
return parent.id
def batch_create_terminology(session: SessionDep, info_list: List[TerminologyInfo], oid: int, trans: Trans):
"""
批量创建术语记录（复用单条插入逻辑）
"""
if not info_list:
return {
'success_count': 0,
'failed_records': [],
'duplicate_count': 0,
'original_count': 0,
'deduplicated_count': 0
}
failed_records = []
success_count = 0
inserted_ids = []
# 第一步：数据去重（根据新的唯一性规则）
unique_records = {}
duplicate_records = []
for info in info_list:
# 过滤掉空的其他词
filtered_other_words = [w.strip().lower() for w in info.other_words if w and w.strip()]
# 根据specific_ds决定是否处理datasource_names
specific_ds = info.specific_ds if info.specific_ds is not None else False
filtered_datasource_names = []
if specific_ds and info.datasource_names:
# 只有当specific_ds为True时才考虑数据源名称
filtered_datasource_names = [d.strip().lower() for d in info.datasource_names if d and d.strip()]
# 创建唯一标识（根据新的规则）
# 1. word和other_words合并并排序（考虑顺序不同）
all_words = [info.word.strip().lower()] if info.word else []
all_words.extend(filtered_other_words)
all_words_sorted = sorted(all_words)
# 2. datasource_names排序（考虑顺序不同）
datasource_names_sorted = sorted(filtered_datasource_names)
unique_key = (
','.join(all_words_sorted), # 合并后的所有词（排序）
','.join(datasource_names_sorted), # 数据源名称（排序）
str(specific_ds) # specific_ds状态
)
if unique_key in unique_records:
duplicate_records.append(info)
else:
unique_records[unique_key] = info
# 将去重后的数据转换为列表
deduplicated_list = list(unique_records.values())
# 预加载数据源名称到ID的映射
datasource_name_to_id = {}
datasource_stmt = select(CoreDatasource.id, CoreDatasource.name).where(CoreDatasource.oid == oid)
datasource_result = session.execute(datasource_stmt).all()
for ds in datasource_result:
datasource_name_to_id[ds.name.strip()] = ds.id
# 验证和转换数据源名称
valid_records = []
for info in deduplicated_list:
error_messages = []
# 基本验证
if not info.word or not info.word.strip():
error_messages.append(trans("i18n_terminology.word_cannot_be_empty"))
failed_records.append({
'data': info,
'errors': error_messages
})
continue
if not info.description or not info.description.strip():
error_messages.append(trans("i18n_terminology.description_cannot_be_empty"))
failed_records.append({
'data': info,
'errors': error_messages
})
continue
# 根据specific_ds决定是否验证数据源
specific_ds = info.specific_ds if info.specific_ds is not None else False
datasource_ids = []
if specific_ds:
# specific_ds为True时需要验证数据源
if info.datasource_names:
for ds_name in info.datasource_names:
if not ds_name or not ds_name.strip():
continue # 跳过空的数据源名称
if ds_name.strip() in datasource_name_to_id:
datasource_ids.append(datasource_name_to_id[ds_name.strip()])
else:
error_messages.append(trans("i18n_terminology.datasource_not_found").format(ds_name))
# 检查specific_ds为True时必须有数据源
if not datasource_ids:
error_messages.append(trans("i18n_terminology.datasource_cannot_be_none"))
else:
# specific_ds为False时忽略数据源名称
datasource_ids = []
# 检查主词和其他词是否重复（过滤空字符串）
words = [info.word.strip().lower()]
if info.other_words:
for other_word in info.other_words:
# 先检查是否为空字符串
if not other_word or other_word.strip() == "":
continue
word_lower = other_word.strip().lower()
if word_lower in words:
error_messages.append(trans("i18n_terminology.cannot_be_repeated"))
else:
words.append(word_lower)
if error_messages:
failed_records.append({
'data': info,
'errors': error_messages
})
continue
# 创建新的TerminologyInfo对象
processed_info = TerminologyInfo(
word=info.word.strip(),
description=info.description.strip(),
other_words=[w for w in info.other_words if w and w.strip()], # 过滤空字符串
datasource_ids=datasource_ids,
datasource_names=info.datasource_names,
specific_ds=specific_ds,
enabled=info.enabled if info.enabled is not None else True
)
valid_records.append(processed_info)
# 使用事务批量处理有效记录
if valid_records:
for info in valid_records:
try:
# 直接复用create_terminology方法
terminology_id = create_terminology(session, info, oid, trans)
inserted_ids.append(terminology_id)
success_count += 1
except Exception as e:
# 如果单条插入失败，回滚当前记录
session.rollback()
failed_records.append({
'data': info,
'errors': [str(e)]
})
return {
'success_count': success_count,
'failed_records': failed_records,
'duplicate_count': len(duplicate_records),
'original_count': len(info_list),
'deduplicated_count': len(deduplicated_list)
}
def update_terminology(session: SessionDep, info: TerminologyInfo, oid: int, trans: Trans):
backend/locales/en.json
+4 -1
View File
@@ -47,7 +47,10 @@
"term_description": "Terminology Description",
"effective_data_sources": "Effective Data Sources",
"all_data_sources": "All Data Sources",
"synonyms": "Synonyms"
"synonyms": "Synonyms",
"word_cannot_be_empty": "Term cannot be empty",
"description_cannot_be_empty": "Term description cannot be empty",
"datasource_not_found": "Datasource not found"
},
"i18n_data_training": {
"datasource_cannot_be_none": "Datasource cannot be empty",
backend/locales/ko-KR.json
+4 -1
View File
@@ -47,7 +47,10 @@
"term_description": "용어 설명",
"effective_data_sources": "유효 데이터 소스",
"all_data_sources": "모든 데이터 소스",
"synonyms": "동의어"
"synonyms": "동의어",
"word_cannot_be_empty": "용어는 비울 수 없습니다",
"description_cannot_be_empty": "용어 설명은 비울 수 없습니다",
"datasource_not_found": "데이터 소스를 찾을 수 없음"
},
"i18n_data_training": {
"datasource_cannot_be_none": "데이터 소스는 비울 수 없습니다",
backend/locales/zh-CN.json
+4 -1
View File
@@ -47,7 +47,10 @@
"term_description": "术语描述",
"effective_data_sources": "生效数据源",
"all_data_sources": "所有数据源",
"synonyms": "同义词"
"synonyms": "同义词",
"word_cannot_be_empty": "术语不能为空",
"description_cannot_be_empty": "术语描述不能为空",
"datasource_not_found": "找不到数据源"
},
"i18n_data_training": {
"datasource_cannot_be_none": "数据源不能为空",