feat: implement LLM keyword extraction and fusion scoring for table matching

- Add extract_keywords() to LLMService with conversation context support,
  using process_stream + start_log/end_log for logging consistency
- Add expand_with_terminology() to expand keywords with synonym matching
- Add fusion scoring in calc_table_embedding(): α * vec_score + (1-α) * keyword_score
- Add calc_keyword_score() with multi-level matching:
  table name exact (1.0), comment exact (0.9), comment substring (0.7),
  table name partial (0.25-0.5), field comment exact (0.5),
  field comment substring (0.4), field name partial (0.3)
- Scoring considers keyword coverage rate to differentiate tables
  with different numbers of matched keywords
- Extracted keywords persisted to ChatRecord for reuse across endpoints
- get_terminology_template() supports comma-separated keyword input
  with per-keyword matching for higher precision
- Add TABLE_EMBEDDING_ALPHA and TABLE_EMBEDDING_KEYWORD_ENABLED config
This commit is contained in:
ulleo
2026-09-14 17:51:45 +08:00
committed by ulleo
parent aac200aef5
commit 3fd6fa867d
15 changed files with 1200 additions and 29 deletions
+4 -4
View File
@@ -24,14 +24,14 @@ config = context.config
# from apps.system.models.user import SQLModel # noqa
# from apps.settings.models.setting_models import SQLModel
#from apps.chat.models.chat_model import SQLModel
from apps.terminology.models.terminology_model import SQLModel
from sqlbot_xpack.custom_prompt.models.custom_prompt_model import SQLModel
from apps.chat.models.chat_model import SQLModel
#from apps.terminology.models.terminology_model import SQLModel
#from sqlbot_xpack.custom_prompt.models.custom_prompt_model import SQLModel
#from apps.data_training.models.data_training_model import SQLModel
# from apps.dashboard.models.dashboard_model import SQLModel
from common.core.config import settings # noqa
#from apps.datasource.models.datasource import SQLModel
from apps.system.models.system_model import SQLModel
#from apps.system.models.system_model import SQLModel
target_metadata = SQLModel.metadata
@@ -0,0 +1,31 @@
"""072_chat_record
Revision ID: f898322341a5
Revises: a2e2ecfa5a9c
Create Date: 2026-09-14 15:54:47.921806
"""
from alembic import op
import sqlalchemy as sa
import sqlmodel.sql.sqltypes
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = 'f898322341a5'
down_revision = 'a2e2ecfa5a9c'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('chat_record', sa.Column('extracted_keywords', sa.Text(), nullable=True, comment='关键词'))
op.add_column('chat_record', sa.Column('expanded_keywords', sa.Text(), nullable=True, comment='扩展后的关键词'))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('chat_record', 'extracted_keywords')
op.drop_column('chat_record', 'expanded_keywords')
# ### end Alembic commands ###