SQLBot/backend/apps/terminology/curd/terminology.py

import datetime
import traceback
from typing import List, Optional, Any
from xml.dom.minidom import parseString

from sqlalchemy import and_, or_, select, func, delete, update, union, text, BigInteger
from sqlalchemy.orm import aliased

from apps.ai_model.embedding import EmbeddingModelCache
from apps.datasource.models.datasource import CoreDatasource
from apps.system.models.system_model import AssistantModel
from apps.template.generate_chart.generator import get_base_terminology_template
from apps.terminology.models.terminology_model import Terminology, TerminologyInfo, TerminologyInfoResult
from common.core.config import settings
from common.core.deps import SessionDep, Trans
from common.utils.dict_to_xml import dict_to_xml
from common.utils.embedding_threads import run_save_terminology_embeddings


def get_terminology_base_query(oid: int, name: Optional[str] = None):
    """
    获取术语查询的基础查询结构
    """
    child = aliased(Terminology)

    if name and name.strip() != "":
        keyword_pattern = f"%{name.strip()}%"
        # 步骤1：先找到所有匹配的节点ID（无论是父节点还是子节点）
        matched_ids_subquery = (
            select(Terminology.id)
            .where(and_(Terminology.word.ilike(keyword_pattern), Terminology.oid == oid))
            .subquery()
        )

        # 步骤2：找到这些匹配节点的所有父节点（包括自身如果是父节点）
        parent_ids_subquery = (
            select(Terminology.id)
            .where(
                (Terminology.id.in_(matched_ids_subquery)) |
                (Terminology.id.in_(
                    select(Terminology.pid)
                    .where(Terminology.id.in_(matched_ids_subquery))
                    .where(Terminology.pid.isnot(None))
                ))
            )
            .where(Terminology.pid.is_(None))  # 只取父节点
        )
    else:
        parent_ids_subquery = (
            select(Terminology.id)
            .where(and_(Terminology.pid.is_(None), Terminology.oid == oid))
        )

    return parent_ids_subquery, child


def build_terminology_query(session: SessionDep, oid: int, name: Optional[str] = None,
                            paginate: bool = True, current_page: int = 1, page_size: int = 10,
                            ds_list: Optional[list[int]] = None, adv_list: Optional[list[int]] = None):
    """
    构建术语查询的通用方法
    """
    parent_ids_subquery, child = get_terminology_base_query(oid, name)

    # 添加数据源/高级应用筛选条件（ds_list 与 adv_list 为或的关系）
    ds_filter_condition = None
    adv_filter_condition = None

    if ds_list is not None and len(ds_list) > 0:
        datasource_conditions = []
        for ds_id in ds_list:
            datasource_conditions.append(
                Terminology.datasource_ids.contains([ds_id])
            )

        ds_filter_condition = or_(
            Terminology.specific_ds == False,
            and_(
                Terminology.specific_ds == True,
                *datasource_conditions
            )
        )

    if adv_list is not None and len(adv_list) > 0:
        adv_filter_condition = Terminology.advanced_application.in_(adv_list)

    if ds_filter_condition is not None and adv_filter_condition is not None:
        parent_ids_subquery = parent_ids_subquery.where(
            and_(ds_filter_condition, adv_filter_condition)
        )
    elif ds_filter_condition is not None:
        parent_ids_subquery = parent_ids_subquery.where(ds_filter_condition)
    elif adv_filter_condition is not None:
        parent_ids_subquery = parent_ids_subquery.where(adv_filter_condition)

    # 计算总数
    count_stmt = select(func.count()).select_from(parent_ids_subquery.subquery())
    total_count = session.execute(count_stmt).scalar()

    if paginate:
        # 分页处理
        page_size = max(10, page_size)
        total_pages = (total_count + page_size - 1) // page_size
        current_page = max(1, min(current_page, total_pages)) if total_pages > 0 else 1

        paginated_parent_ids = (
            parent_ids_subquery
            .order_by(Terminology.create_time.desc())
            .offset((current_page - 1) * page_size)
            .limit(page_size)
            .subquery()
        )
    else:
        # 不分页，获取所有数据
        total_pages = 1
        current_page = 1
        page_size = total_count if total_count > 0 else 1

        paginated_parent_ids = (
            parent_ids_subquery
            .order_by(Terminology.create_time.desc())
            .subquery()
        )

    # 构建公共查询部分
    children_subquery = (
        select(
            child.pid,
            func.jsonb_agg(child.word).filter(child.word.isnot(None)).label('other_words')
        )
        .where(child.pid.isnot(None))
        .group_by(child.pid)
        .subquery()
    )

    # 创建子查询来获取数据源名称
    datasource_names_subquery = (
        select(
            func.jsonb_array_elements(Terminology.datasource_ids).cast(BigInteger).label('ds_id'),
            Terminology.id.label('term_id')
        )
        .where(Terminology.id.in_(paginated_parent_ids))
        .subquery()
    )

    stmt = (
        select(
            Terminology.id,
            Terminology.word,
            Terminology.create_time,
            Terminology.description,
            Terminology.specific_ds,
            Terminology.datasource_ids,
            children_subquery.c.other_words,
            func.jsonb_agg(CoreDatasource.name).filter(CoreDatasource.id.isnot(None)).label('datasource_names'),
            Terminology.enabled,
            Terminology.advanced_application,
            AssistantModel.name.label('advanced_application_name'),
        )
        .outerjoin(
            children_subquery,
            Terminology.id == children_subquery.c.pid
        )
        .outerjoin(
            datasource_names_subquery,
            datasource_names_subquery.c.term_id == Terminology.id
        )
        .outerjoin(
            CoreDatasource,
            CoreDatasource.id == datasource_names_subquery.c.ds_id
        )
        .outerjoin(AssistantModel,
                   and_(Terminology.advanced_application == AssistantModel.id, AssistantModel.type == 1))
        .where(and_(Terminology.id.in_(paginated_parent_ids), Terminology.oid == oid))
        .group_by(
            Terminology.id,
            Terminology.word,
            Terminology.create_time,
            Terminology.description,
            Terminology.specific_ds,
            Terminology.datasource_ids,
            Terminology.advanced_application,
            AssistantModel.name,
            children_subquery.c.other_words,
            Terminology.enabled
        )
        .order_by(Terminology.create_time.desc())
    )

    return stmt, total_count, total_pages, current_page, page_size


def execute_terminology_query(session: SessionDep, stmt) -> List[TerminologyInfoResult]:
    """
    执行查询并返回术语信息列表
    """
    _list = []
    result = session.execute(stmt)

    for row in result:
        _list.append(TerminologyInfoResult(
            id=row.id,
            word=row.word,
            create_time=row.create_time,
            description=row.description,
            other_words=row.other_words if row.other_words else [],
            specific_ds=row.specific_ds if row.specific_ds is not None else False,
            datasource_ids=row.datasource_ids if row.datasource_ids is not None else [],
            datasource_names=row.datasource_names if row.datasource_names is not None else [],
            enabled=row.enabled if row.enabled is not None else False,
            advanced_application=str(row.advanced_application) if row.advanced_application else None,
            advanced_application_name=row.advanced_application_name,
        ))

    return _list


def page_terminology(session: SessionDep, current_page: int = 1, page_size: int = 10,
                     name: Optional[str] = None, oid: Optional[int] = 1, ds_list: Optional[list[int]] = None,
                     adv_list: Optional[list[int]] = None):
    """
    分页查询术语（原方法保持不变）
    """
    stmt, total_count, total_pages, current_page, page_size = build_terminology_query(
        session, oid, name, True, current_page, page_size, ds_list, adv_list
    )
    _list = execute_terminology_query(session, stmt)

    return current_page, page_size, total_count, total_pages, _list


def get_all_terminology(session: SessionDep, name: Optional[str] = None, oid: Optional[int] = 1,
                        ds_list: Optional[list[int]] = None, adv_list: Optional[list[int]] = None):
    """
    获取所有术语（不分页）
    """
    stmt, total_count, total_pages, current_page, page_size = build_terminology_query(
        session, oid, name, False, ds_list=ds_list, adv_list=adv_list
    )
    _list = execute_terminology_query(session, stmt)

    return _list


def create_terminology(session: SessionDep, info: TerminologyInfo, oid: int, trans: Trans,
                       skip_embedding: bool = False):
    """
    创建单个术语记录
    Args:
        skip_embedding: 是否跳过embedding处理（用于批量插入）
    """
    # 基本验证
    if not info.word or not info.word.strip():
        raise Exception(trans("i18n_terminology.word_cannot_be_empty"))

    if not info.description or not info.description.strip():
        raise Exception(trans("i18n_terminology.description_cannot_be_empty"))

    create_time = datetime.datetime.now()

    specific_ds = info.specific_ds if info.specific_ds is not None else False
    datasource_ids = info.datasource_ids if info.datasource_ids is not None else []

    if specific_ds:
        if not datasource_ids and info.advanced_application is None:
            raise Exception(trans("i18n_data_training.datasource_assistant_cannot_be_none"))

    parent = Terminology(
        word=info.word.strip(),
        create_time=create_time,
        description=info.description.strip(),
        oid=oid,
        specific_ds=specific_ds,
        enabled=info.enabled,
        datasource_ids=datasource_ids,
        advanced_application=info.advanced_application,
    )

    words = [info.word.strip()]
    for child_word in info.other_words:
        # 先检查是否为空字符串
        if not child_word or child_word.strip() == "":
            continue

        if child_word in words:
            raise Exception(trans("i18n_terminology.cannot_be_repeated"))
        else:
            words.append(child_word.strip())

    # 基础查询条件（word 和 oid 必须满足）
    base_query = and_(
        Terminology.word.in_(words),
        Terminology.oid == oid,
    )

    # 构建查询
    query = session.query(Terminology).filter(base_query)

    # 作用域重复检查
    scope_conditions = []

    if not specific_ds:
        # 全部数据源：与有数据源的记录冲突，也与同为全部数据源的记录冲突
        scope_conditions.append(
            and_(
                Terminology.specific_ds == True,
                Terminology.datasource_ids.isnot(None),
                func.jsonb_array_length(Terminology.datasource_ids) > 0,
            )
        )
        scope_conditions.append(
            Terminology.specific_ds == False,
        )
    elif specific_ds and datasource_ids:
        # 指定数据源：与全部数据源冲突，也与有数据源重叠的记录冲突
        scope_conditions.append(
            Terminology.specific_ds == False,
        )
        ds_overlap_conditions = [
            Terminology.datasource_ids.contains([ds_id])
            for ds_id in datasource_ids
        ]
        scope_conditions.append(
            and_(
                Terminology.specific_ds == True,
                Terminology.datasource_ids.isnot(None),
                or_(*ds_overlap_conditions)
            )
        )
    else:
        # 不选数据源：仅与同为不选数据源的记录冲突
        scope_conditions.append(
            and_(
                Terminology.specific_ds == True,
                or_(
                    Terminology.datasource_ids.is_(None),
                    func.jsonb_array_length(Terminology.datasource_ids) == 0,
                )
            )
        )

    # 高级应用重复检查：advanced_application 相同时同名即重复
    if info.advanced_application is not None:
        scope_conditions.append(
            Terminology.advanced_application == info.advanced_application
        )

    if scope_conditions:
        query = query.where(or_(*scope_conditions))

    # 转换为 EXISTS 查询并获取结果
    exists = session.query(query.exists()).scalar()

    if exists:
        raise Exception(trans("i18n_terminology.exists_in_db"))

    session.add(parent)
    session.flush()
    session.refresh(parent)

    # 插入子记录（其他词）
    child_list = []
    if info.other_words:
        for other_word in info.other_words:
            if other_word.strip() == "":
                continue
            child_list.append(
                Terminology(
                    pid=parent.id,
                    word=other_word.strip(),
                    create_time=create_time,
                    oid=oid,
                    enabled=info.enabled,
                    specific_ds=specific_ds,
                    datasource_ids=datasource_ids
                )
            )

    if child_list:
        session.bulk_save_objects(child_list)
        session.flush()

    session.commit()

    # 处理embedding（批量插入时跳过）
    if not skip_embedding:
        run_save_terminology_embeddings([parent.id])

    return parent.id


def batch_create_terminology(session: SessionDep, info_list: List[TerminologyInfo], oid: int, trans: Trans):
    """
    批量创建术语记录（复用单条插入逻辑）
    """
    if not info_list:
        return {
            'success_count': 0,
            'failed_records': [],
            'duplicate_count': 0,
            'original_count': 0,
            'deduplicated_count': 0
        }

    failed_records = []
    success_count = 0
    inserted_ids = []

    # 第一步：数据去重（根据新的唯一性规则）
    unique_records = {}
    duplicate_records = []

    for info in info_list:
        # 过滤掉空的其他词
        filtered_other_words = [w.strip().lower() for w in info.other_words if w and w.strip()]

        # 根据specific_ds决定是否处理datasource_names
        specific_ds = info.specific_ds if info.specific_ds is not None else False
        filtered_datasource_names = []

        if specific_ds and info.datasource_names:
            # 只有当specific_ds为True时才考虑数据源名称
            filtered_datasource_names = [d.strip().lower() for d in info.datasource_names if d and d.strip()]

        # 创建唯一标识（根据新的规则）
        # 1. word和other_words合并并排序（考虑顺序不同）
        all_words = [info.word.strip().lower()] if info.word else []
        all_words.extend(filtered_other_words)
        all_words_sorted = sorted(all_words)

        # 2. datasource_names排序（考虑顺序不同）
        datasource_names_sorted = sorted(filtered_datasource_names)

        unique_key = (
            ','.join(all_words_sorted),  # 合并后的所有词（排序）
            ','.join(datasource_names_sorted),  # 数据源名称（排序）
            str(specific_ds)  # specific_ds状态
        )

        if unique_key in unique_records:
            duplicate_records.append(info)
        else:
            unique_records[unique_key] = info

    # 将去重后的数据转换为列表
    deduplicated_list = list(unique_records.values())

    # 预加载数据源名称到ID的映射
    datasource_name_to_id = {}
    datasource_stmt = select(CoreDatasource.id, CoreDatasource.name).where(CoreDatasource.oid == oid)
    datasource_result = session.execute(datasource_stmt).all()
    for ds in datasource_result:
        datasource_name_to_id[ds.name.strip()] = ds.id

    # 预加载高级应用名称到ID的映射
    assistant_name_to_id = {}
    assistant_stmt = select(AssistantModel.id, AssistantModel.name).where(
        and_(AssistantModel.oid == oid, AssistantModel.type == 1))
    assistant_result = session.execute(assistant_stmt).all()
    for a in assistant_result:
        assistant_name_to_id[a.name.strip()] = a.id

    # 验证和转换数据源名称
    valid_records = []
    for info in deduplicated_list:
        error_messages = []

        # 基本验证
        if not info.word or not info.word.strip():
            error_messages.append(trans("i18n_terminology.word_cannot_be_empty"))

        if not info.description or not info.description.strip():
            error_messages.append(trans("i18n_terminology.description_cannot_be_empty"))

        # 根据specific_ds决定是否验证数据源
        specific_ds = info.specific_ds if info.specific_ds is not None else False
        datasource_ids = []
        advanced_application = info.advanced_application

        if specific_ds:
            # specific_ds为True时需要验证数据源
            if info.datasource_names:
                for ds_name in info.datasource_names:
                    if not ds_name or not ds_name.strip():
                        continue  # 跳过空的数据源名称

                    if ds_name.strip() in datasource_name_to_id:
                        datasource_ids.append(datasource_name_to_id[ds_name.strip()])
                    else:
                        error_messages.append(trans("i18n_terminology.datasource_not_found").format(ds_name))

            # 解析高级应用名称到ID
            if advanced_application is None and info.advanced_application_name:
                if info.advanced_application_name.strip() in assistant_name_to_id:
                    advanced_application = assistant_name_to_id[info.advanced_application_name.strip()]
                else:
                    error_messages.append(trans("i18n_data_training.advanced_application_not_found").format(
                        info.advanced_application_name))

            # 检查specific_ds为True时datasource_ids和advanced_application不能同时为空
            if specific_ds and not datasource_ids and advanced_application is None:
                error_messages.append(trans("i18n_data_training.datasource_assistant_cannot_be_none"))
        else:
            # specific_ds为False时忽略数据源名称
            datasource_ids = []
            advanced_application = None

        # 检查主词和其他词是否重复（过滤空字符串）
        words = [info.word.strip().lower()]
        if info.other_words:
            for other_word in info.other_words:
                # 先检查是否为空字符串
                if not other_word or other_word.strip() == "":
                    continue

                word_lower = other_word.strip().lower()
                if word_lower in words:
                    error_messages.append(trans("i18n_terminology.cannot_be_repeated"))
                else:
                    words.append(word_lower)

        if error_messages:
            failed_records.append({
                'data': info,
                'errors': error_messages
            })
            continue

        # 创建新的TerminologyInfo对象
        processed_info = TerminologyInfo(
            word=info.word.strip(),
            description=info.description.strip(),
            other_words=[w for w in info.other_words if w and w.strip()],
            datasource_ids=datasource_ids,
            datasource_names=info.datasource_names,
            specific_ds=specific_ds,
            enabled=info.enabled if info.enabled is not None else True,
            advanced_application=advanced_application,
            advanced_application_name=info.advanced_application_name,
        )

        valid_records.append(processed_info)

    # 使用事务批量处理有效记录
    if valid_records:
        for info in valid_records:
            try:
                # 直接复用create_terminology方法，跳过embedding处理
                terminology_id = create_terminology(session, info, oid, trans, skip_embedding=True)
                inserted_ids.append(terminology_id)
                success_count += 1

            except Exception as e:
                # 如果单条插入失败，回滚当前记录
                session.rollback()
                failed_records.append({
                    'data': info,
                    'errors': [str(e)]
                })

        # 批量处理embedding（只在最后执行一次）
        if success_count > 0 and inserted_ids:
            try:
                run_save_terminology_embeddings(inserted_ids)
            except Exception as e:
                # 如果embedding处理失败，记录错误但不回滚数据
                print(f"Terminology embedding processing failed: {str(e)}")
                # 可以选择将embedding失败的信息记录到日志或返回给调用方

    return {
        'success_count': success_count,
        'failed_records': failed_records,
        'duplicate_count': len(duplicate_records),
        'original_count': len(info_list),
        'deduplicated_count': len(deduplicated_list)
    }


def update_terminology(session: SessionDep, info: TerminologyInfo, oid: int, trans: Trans):
    count = session.query(Terminology).filter(
        Terminology.oid == oid,
        Terminology.id == info.id
    ).count()
    if count == 0:
        raise Exception(trans('i18n_terminology.terminology_not_exists'))

    specific_ds = info.specific_ds if info.specific_ds is not None else False
    datasource_ids = info.datasource_ids if info.datasource_ids is not None else []

    if specific_ds:
        if not datasource_ids and info.advanced_application is None:
            raise Exception(trans("i18n_data_training.datasource_assistant_cannot_be_none"))

    words = [info.word.strip()]
    for child in info.other_words:
        if child in words:
            raise Exception(trans("i18n_terminology.cannot_be_repeated"))
        else:
            words.append(child.strip())

    # 基础查询条件（word 和 oid 必须满足）
    base_query = and_(
        Terminology.word.in_(words),
        Terminology.oid == oid,
        or_(
            Terminology.pid != info.id,
            and_(Terminology.pid.is_(None), Terminology.id != info.id)
        ),
        Terminology.id != info.id
    )

    # 构建查询
    query = session.query(Terminology).filter(base_query)

    # 作用域重复检查
    scope_conditions = []

    if not specific_ds:
        # 全部数据源：与有数据源的记录冲突，也与同为全部数据源的记录冲突
        scope_conditions.append(
            and_(
                Terminology.specific_ds == True,
                Terminology.datasource_ids.isnot(None),
                func.jsonb_array_length(Terminology.datasource_ids) > 0,
            )
        )
        scope_conditions.append(
            Terminology.specific_ds == False,
        )
    elif specific_ds and datasource_ids:
        # 指定数据源：与全部数据源冲突，也与有数据源重叠的记录冲突
        scope_conditions.append(
            Terminology.specific_ds == False,
        )
        ds_overlap_conditions = [
            Terminology.datasource_ids.contains([ds_id])
            for ds_id in datasource_ids
        ]
        scope_conditions.append(
            and_(
                Terminology.specific_ds == True,
                Terminology.datasource_ids.isnot(None),
                or_(*ds_overlap_conditions)
            )
        )
    else:
        # 不选数据源：仅与同为不选数据源的记录冲突
        scope_conditions.append(
            and_(
                Terminology.specific_ds == True,
                or_(
                    Terminology.datasource_ids.is_(None),
                    func.jsonb_array_length(Terminology.datasource_ids) == 0,
                )
            )
        )

    # 高级应用重复检查：advanced_application 相同时同名即重复
    if info.advanced_application is not None:
        scope_conditions.append(
            Terminology.advanced_application == info.advanced_application
        )

    if scope_conditions:
        query = query.where(or_(*scope_conditions))

    # 转换为 EXISTS 查询并获取结果
    exists = session.query(query.exists()).scalar()

    if exists:
        raise Exception(trans("i18n_terminology.exists_in_db"))

    stmt = update(Terminology).where(and_(Terminology.id == info.id)).values(
        word=info.word.strip(),
        description=info.description.strip(),
        specific_ds=specific_ds,
        datasource_ids=datasource_ids,
        enabled=info.enabled,
        advanced_application=info.advanced_application,
    )
    session.execute(stmt)
    session.commit()

    stmt = delete(Terminology).where(and_(Terminology.pid == info.id))
    session.execute(stmt)
    session.commit()

    create_time = datetime.datetime.now()
    # 插入子记录（其他词）
    child_list = []
    if info.other_words:
        for other_word in info.other_words:
            if other_word.strip() == "":
                continue
            child_list.append(
                Terminology(
                    pid=info.id,
                    word=other_word.strip(),
                    create_time=create_time,
                    oid=oid,
                    enabled=info.enabled,
                    specific_ds=specific_ds,
                    datasource_ids=datasource_ids,
                    advanced_application=info.advanced_application,
                )
            )

    if child_list:
        session.bulk_save_objects(child_list)
        session.flush()
    session.commit()

    # embedding
    run_save_terminology_embeddings([info.id])

    return info.id


def delete_terminology(session: SessionDep, ids: list[int], oid: int):
    stmt = delete(Terminology).where(
        and_(Terminology.oid == oid, or_(Terminology.id.in_(ids), Terminology.pid.in_(ids))))
    session.execute(stmt)
    session.commit()


def enable_terminology(session: SessionDep, id: int, enabled: bool, trans: Trans, oid: int):
    count = session.query(Terminology).filter(
        Terminology.id == id,
        Terminology.oid == oid,
    ).count()
    if count == 0:
        raise Exception(trans('i18n_terminology.terminology_not_exists'))

    stmt = update(Terminology).where(
        and_(Terminology.oid == oid, or_(Terminology.id == id, Terminology.pid == id))).values(
        enabled=enabled,
    )
    session.execute(stmt)
    session.commit()


# def run_save_embeddings(ids: List[int]):
#     executor.submit(save_embeddings, ids)
#
#
# def fill_empty_embeddings():
#     executor.submit(run_fill_empty_embeddings)
# from sqlalchemy import create_engine
# from sqlalchemy.orm import sessionmaker,scoped_session
# engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI))
# session_maker = scoped_session(sessionmaker(bind=engine))

def run_fill_empty_embeddings(session_maker):
    try:
        if not settings.EMBEDDING_ENABLED:
            return
        session = session_maker()
        stmt1 = select(Terminology.id).where(and_(Terminology.embedding.is_(None), Terminology.pid.is_(None)))
        stmt2 = select(Terminology.pid).where(
            and_(Terminology.embedding.is_(None), Terminology.pid.isnot(None))).distinct()
        combined_stmt = union(stmt1, stmt2)
        results = session.execute(combined_stmt).scalars().all()
        save_embeddings(session_maker, results)
    except Exception:
        traceback.print_exc()
    finally:
        session_maker.remove()


def save_embeddings(session_maker, ids: List[int]):
    if not settings.EMBEDDING_ENABLED:
        return

    if not ids or len(ids) == 0:
        return
    try:
        session = session_maker()
        _list = session.query(Terminology).filter(or_(Terminology.id.in_(ids), Terminology.pid.in_(ids))).all()

        _words_list = [item.word for item in _list]

        model = EmbeddingModelCache.get_model()

        results = model.embed_documents(_words_list)

        for index in range(len(results)):
            item = results[index]
            stmt = update(Terminology).where(and_(Terminology.id == _list[index].id)).values(embedding=item)
            session.execute(stmt)
            session.commit()

    except Exception:
        traceback.print_exc()
    finally:
        session_maker.remove()


embedding_sql = f"""
SELECT id, pid, word, similarity
FROM
(SELECT id, pid, word, oid, specific_ds, datasource_ids, enabled,
( 1 - (embedding <=> :embedding_array) ) AS similarity
FROM terminology AS child
) TEMP
WHERE similarity > {settings.EMBEDDING_TERMINOLOGY_SIMILARITY} AND oid = :oid AND enabled = true
AND (specific_ds = false OR specific_ds IS NULL)
ORDER BY similarity DESC
LIMIT {settings.EMBEDDING_TERMINOLOGY_TOP_COUNT}
"""

embedding_sql_with_datasource = f"""
SELECT id, pid, word, similarity
FROM
(SELECT id, pid, word, oid, specific_ds, datasource_ids, enabled,
( 1 - (embedding <=> :embedding_array) ) AS similarity
FROM terminology AS child
) TEMP
WHERE similarity > {settings.EMBEDDING_TERMINOLOGY_SIMILARITY} AND oid = :oid AND enabled = true
AND (
    (specific_ds = false OR specific_ds IS NULL)
     OR
    (specific_ds = true AND datasource_ids IS NOT NULL AND datasource_ids @> jsonb_build_array(:datasource))
)
ORDER BY similarity DESC
LIMIT {settings.EMBEDDING_TERMINOLOGY_TOP_COUNT}
"""

embedding_sql_with_advanced_application = f"""
SELECT id, pid, word, similarity
FROM
(SELECT id, pid, word, oid, specific_ds, advanced_application, enabled,
( 1 - (embedding <=> :embedding_array) ) AS similarity
FROM terminology AS child
) TEMP
WHERE similarity > {settings.EMBEDDING_TERMINOLOGY_SIMILARITY} AND oid = :oid AND enabled = true
AND advanced_application = :advanced_application_id
ORDER BY similarity DESC
LIMIT {settings.EMBEDDING_TERMINOLOGY_TOP_COUNT}
"""


def select_terminology_by_word(session: SessionDep, word: str, oid: int, datasource: int = None,
                               advanced_application_id: Optional[int] = None):
    if word.strip() == "":
        return []

    _list: List[Terminology] = []

    stmt = (
        select(
            Terminology.id,
            Terminology.pid,
            Terminology.word,
        )
        .where(
            and_(text(":sentence ILIKE '%' || word || '%'"), Terminology.oid == oid, Terminology.enabled == True)
        )
    )

    if advanced_application_id is not None:
        stmt = stmt.where(Terminology.advanced_application == advanced_application_id)
    elif datasource is not None:
        stmt = stmt.where(
            or_(
                or_(Terminology.specific_ds == False, Terminology.specific_ds.is_(None)),
                and_(
                    Terminology.specific_ds == True,
                    Terminology.datasource_ids.isnot(None),
                    text("datasource_ids @> jsonb_build_array(:datasource)")
                )
            )
        )
    else:
        stmt = stmt.where(or_(Terminology.specific_ds == False, Terminology.specific_ds.is_(None)))

    # 执行查询
    params: dict[str, Any] = {'sentence': word}
    if datasource is not None:
        params['datasource'] = datasource

    results = session.execute(stmt, params).fetchall()

    for row in results:
        _list.append(Terminology(id=row.id, word=row.word, pid=row.pid))

    if settings.EMBEDDING_ENABLED:
        with session.begin_nested():
            try:
                model = EmbeddingModelCache.get_model()

                embedding = model.embed_query(word)

                if advanced_application_id is not None:
                    results = session.execute(text(embedding_sql_with_advanced_application),
                                              {'embedding_array': str(embedding), 'oid': oid,
                                               'advanced_application_id': advanced_application_id}).fetchall()
                elif datasource is not None:
                    results = session.execute(text(embedding_sql_with_datasource),
                                              {'embedding_array': str(embedding), 'oid': oid,
                                               'datasource': datasource}).fetchall()
                else:
                    results = session.execute(text(embedding_sql),
                                              {'embedding_array': str(embedding), 'oid': oid}).fetchall()

                for row in results:
                    _list.append(Terminology(id=row.id, word=row.word, pid=row.pid))

            except Exception:
                traceback.print_exc()
                session.rollback()

    _map: dict = {}
    _ids: list[int] = []
    for row in _list:
        if row.id in _ids or row.pid in _ids:
            continue
        if row.pid is not None:
            _ids.append(row.pid)
        else:
            _ids.append(row.id)

    if len(_ids) == 0:
        return []

    t_list = session.query(Terminology.id, Terminology.pid, Terminology.word, Terminology.description).filter(
        or_(Terminology.id.in_(_ids), Terminology.pid.in_(_ids))).all()
    for row in t_list:
        pid = str(row.pid) if row.pid is not None else str(row.id)
        if _map.get(pid) is None:
            _map[pid] = {'words': [], 'description': ''}
        if row.pid is None:
            _map[pid]['description'] = row.description
        _map[pid]['words'].append(row.word)

    _results: list[dict] = []
    for key in _map.keys():
        _results.append(_map.get(key))

    return _results


def get_example():
    _obj = {
        'terminologies': [
            {'words': ['GDP', '国内生产总值'],
             'description': '指在一个季度或一年，一个国家或地区的经济中所生产出的全部最终产品和劳务的价值。'},
        ]
    }
    return to_xml_string(_obj, 'example')


def to_xml_string(_dict: list[dict] | dict, root: str = 'terminologies') -> str:
    item_name_func = lambda x: 'terminology' if x == 'terminologies' else 'word' if x == 'words' else 'item'
    xml = dict_to_xml(_dict, root_name=root, item_func=item_name_func)
    pretty_xml = parseString(xml).toprettyxml()

    if pretty_xml.startswith('<?xml'):
        end_index = pretty_xml.find('>') + 1
        pretty_xml = pretty_xml[end_index:].lstrip()

    # 替换所有 XML 转义字符
    escape_map = {
        '&lt;': '<',
        '&gt;': '>',
        '&amp;': '&',
        '&quot;': '"',
        '&apos;': "'"
    }
    for escaped, original in escape_map.items():
        pretty_xml = pretty_xml.replace(escaped, original)

    return pretty_xml


def get_terminology_template(session: SessionDep, question: str, oid: Optional[int] = 1,
                             datasource: Optional[int] = None,
                             advanced_application_id: Optional[int] = None) -> tuple[str, list[dict]]:
    if not oid:
        oid = 1

    # 有逗号时视为关键词列表，逐个匹配（精度更高，避免整体 ILIKE 误匹配）
    # 无逗号时视为自然语言问题，整体匹配
    if ',' in question:
        parts = [kw.strip() for kw in question.split(',') if kw.strip()]
    else:
        parts = [question]

    # select_terminology_by_word 返回 list[dict]，每个 dict 结构：
    # {'words': ['销售', 'sales'], 'description': '...'}
    seen_words = set()
    _results = []
    for part in parts:
        for item in select_terminology_by_word(session, part, oid, datasource, advanced_application_id):
            item_words = set(item.get('words', [])) if isinstance(item, dict) else set()
            if item_words and not item_words.issubset(seen_words):
                seen_words.update(item_words)
                _results.append(item)

    if _results:
        terminology = to_xml_string(_results)
        template = get_base_terminology_template().format(terminologies=terminology)
        return template, _results
    else:
        return '', []


def expand_with_terminology(keywords: str, session: SessionDep, oid: int) -> str:
    """使用术语同义词扩展关键词。

    查询当前租户启用的术语表，将匹配的关键词扩展为同义词集合。
    pid=None 表示父术语（主词），pid=<id> 表示子术语（同义词）。

    Args:
        keywords: 逗号分隔的关键词字符串
        session: 数据库会话
        oid: 组织 ID

    Returns:
        扩展后的关键词字符串（逗号分隔），最多 10 个关键词
    """
    if not keywords or not session or oid is None:
        return keywords

    keyword_list = [kw.strip() for kw in keywords.split(',') if kw.strip()]
    if not keyword_list:
        return keywords

    try:
        # 查询当前租户启用的术语（限制 500 条避免内存问题）
        terms = session.query(Terminology).filter(
            Terminology.oid == oid,
            Terminology.enabled == True
        ).limit(500).all()

        if not terms:
            return keywords

        # 构建父子结构
        parent_map = {}  # id -> Terminology（仅父术语）
        children_map = {}  # parent_id -> [Terminology]（子术语列表）
        word_to_parent_ids = {}  # 小写词 -> 父术语 id 集合

        for term in terms:
            if term.pid is None or term.pid == 0:
                parent_map[term.id] = term
                if term.word:
                    w = term.word.strip().lower()
                    if w not in word_to_parent_ids:
                        word_to_parent_ids[w] = set()
                    word_to_parent_ids[w].add(term.id)
            else:
                if term.pid not in children_map:
                    children_map[term.pid] = []
                children_map[term.pid].append(term)
                if term.word:
                    w = term.word.strip().lower()
                    if w not in word_to_parent_ids:
                        word_to_parent_ids[w] = set()
                    word_to_parent_ids[w].add(term.pid)

        # 扩展关键词
        expanded = list(keyword_list)
        seen = set(kw.lower() for kw in keyword_list)
        max_keywords = 10

        for kw in keyword_list:
            if len(expanded) >= max_keywords:
                break

            kw_lower = kw.lower()
            matched_parent_ids = word_to_parent_ids.get(kw_lower, set())

            for pid in matched_parent_ids:
                if len(expanded) >= max_keywords:
                    break

                # 添加父术语
                parent = parent_map.get(pid)
                if parent and parent.word:
                    parent_word = parent.word.strip()
                    if parent_word.lower() not in seen:
                        expanded.append(parent_word)
                        seen.add(parent_word.lower())

                # 添加子术语（同义词）
                for child in children_map.get(pid, []):
                    if len(expanded) >= max_keywords:
                        break
                    if child.word:
                        child_word = child.word.strip()
                        if child_word.lower() not in seen:
                            expanded.append(child_word)
                            seen.add(child_word.lower())

        return ','.join(expanded)
    except Exception:
        traceback.print_exc()
        return keywords