mirror of
https://gh-proxy.org/https://github.com/klinsmaya/SQLBot.git
synced 2026-11-08 01:06:59 +08:00
92 lines
4.0 KiB
Python
92 lines
4.0 KiB
Python
# Author: Junjun
|
|
# Date: 2025/9/18
|
|
import json
|
|
import time
|
|
import traceback
|
|
from typing import Optional
|
|
|
|
from apps.ai_model.embedding import EmbeddingModelCache
|
|
from apps.datasource.embedding.utils import cosine_similarity
|
|
from apps.datasource.models.datasource import CoreDatasource
|
|
from apps.system.crud.assistant import AssistantOutDs
|
|
from common.core.config import settings
|
|
from common.core.deps import CurrentAssistant
|
|
from common.core.deps import SessionDep
|
|
from common.utils.utils import SQLBotLogUtil
|
|
|
|
|
|
def get_ds_embedding(session: SessionDep, _ds_list, out_ds: AssistantOutDs,
|
|
question: str,
|
|
current_assistant: Optional[CurrentAssistant] = None):
|
|
_list = []
|
|
if current_assistant and current_assistant.type == 1:
|
|
if out_ds.ds_list:
|
|
for _ds in out_ds.ds_list:
|
|
ds = out_ds.get_ds(_ds.id)
|
|
table_schema, tables = out_ds.get_db_schema(_ds.id, question, embedding=False)
|
|
ds_info = f"{ds.name}, {ds.description}\n"
|
|
ds_schema = ds_info + table_schema
|
|
_list.append({"id": ds.id, "ds_schema": ds_schema, "cosine_similarity": 0.0, "ds": ds})
|
|
|
|
if _list:
|
|
try:
|
|
text = [s.get('ds_schema') for s in _list]
|
|
|
|
model = EmbeddingModelCache.get_model()
|
|
results = model.embed_documents(text)
|
|
|
|
q_embedding = model.embed_query(question)
|
|
for index in range(len(results)):
|
|
item = results[index]
|
|
_list[index]['cosine_similarity'] = cosine_similarity(q_embedding, item)
|
|
|
|
_list.sort(key=lambda x: x['cosine_similarity'], reverse=True)
|
|
# print(len(_list))
|
|
_list = _list[:settings.DS_EMBEDDING_COUNT]
|
|
SQLBotLogUtil.info(json.dumps(
|
|
[{"id": ele.get("id"), "name": ele.get("ds").name,
|
|
"cosine_similarity": ele.get("cosine_similarity")}
|
|
for ele in _list]))
|
|
return [{"id": obj.get('ds').id, "name": obj.get('ds').name, "description": obj.get('ds').description}
|
|
for obj in _list]
|
|
except Exception:
|
|
traceback.print_exc()
|
|
else:
|
|
for _ds in _ds_list:
|
|
if _ds.get('id'):
|
|
ds = session.get(CoreDatasource, _ds.get('id'))
|
|
# table_schema = get_table_schema(session, current_user, ds, question, embedding=False)
|
|
# ds_info = f"{ds.name}, {ds.description}\n"
|
|
# ds_schema = ds_info + table_schema
|
|
_list.append({"id": ds.id, "cosine_similarity": 0.0, "ds": ds, "embedding": ds.embedding})
|
|
|
|
if _list:
|
|
try:
|
|
# text = [s.get('ds_schema') for s in _list]
|
|
|
|
model = EmbeddingModelCache.get_model()
|
|
start_time = time.time()
|
|
# results = model.embed_documents(text)
|
|
results = [item.get('embedding') for item in _list]
|
|
|
|
q_embedding = model.embed_query(question)
|
|
for index in range(len(results)):
|
|
item = results[index]
|
|
if item:
|
|
_list[index]['cosine_similarity'] = cosine_similarity(q_embedding, json.loads(item))
|
|
|
|
_list.sort(key=lambda x: x['cosine_similarity'], reverse=True)
|
|
# print(len(_list))
|
|
end_time = time.time()
|
|
SQLBotLogUtil.info(str(end_time - start_time))
|
|
_list = _list[:settings.DS_EMBEDDING_COUNT]
|
|
SQLBotLogUtil.info(json.dumps(
|
|
[{"id": ele.get("id"), "name": ele.get("ds").name,
|
|
"cosine_similarity": ele.get("cosine_similarity")}
|
|
for ele in _list]))
|
|
return [{"id": obj.get('ds').id, "name": obj.get('ds').name, "description": obj.get('ds').description}
|
|
for obj in _list]
|
|
except Exception:
|
|
traceback.print_exc()
|
|
return _list
|