diff --git a/backend/apps/ai_model/model_factory.py b/backend/apps/ai_model/model_factory.py index 7a65f873..0ba82c42 100644 --- a/backend/apps/ai_model/model_factory.py +++ b/backend/apps/ai_model/model_factory.py @@ -1,3 +1,4 @@ +from functools import lru_cache import json from abc import ABC, abstractmethod from typing import Optional, Dict, Any, Type @@ -21,6 +22,18 @@ class LLMConfig(BaseModel): api_key: Optional[str] = None api_base_url: Optional[str] = None additional_params: Dict[str, Any] = {} + class Config: + frozen = True + + def __hash__(self): + return hash(( + self.model_id, + self.model_type, + self.model_name, + self.api_key, + self.api_base_url, + frozenset(self.additional_params.items()) + )) class BaseLLM(ABC): @@ -66,6 +79,7 @@ class LLMFactory: } @classmethod + @lru_cache(maxsize=32) def create_llm(cls, config: LLMConfig) -> BaseLLM: llm_class = cls._llm_types.get(config.model_type) if not llm_class: diff --git a/backend/apps/system/api/aimodel.py b/backend/apps/system/api/aimodel.py index be5a46b9..d2127cde 100644 --- a/backend/apps/system/api/aimodel.py +++ b/backend/apps/system/api/aimodel.py @@ -27,12 +27,8 @@ async def check_llm(info: AiModelCreator, trans: Trans): additional_params=additional_params, ) llm_instance = LLMFactory.create_llm(config) - - res = llm_instance.llm.stream("who are you?") - - for chunk in res: + async for chunk in llm_instance.llm.astream("1+1=?"): if chunk and chunk.content: - SQLBotLogUtil.info(chunk) yield json.dumps({"content": chunk.content}) + "\n" except Exception as e: diff --git a/frontend/src/views/system/model/Model.vue b/frontend/src/views/system/model/Model.vue index f92a1c3a..6e8e6641 100644 --- a/frontend/src/views/system/model/Model.vue +++ b/frontend/src/views/system/model/Model.vue @@ -35,7 +35,7 @@ const activeName = ref('') const activeType = ref('') const modelFormRef = ref() const cardRefs = ref([]) -// const showCardError = ref(true) // if you don`t want card mask error, just change this to false +const showCardError = ref(false) // if you don`t want card mask error, just change this to false reactive({ form: { id: '', @@ -71,8 +71,6 @@ const defaultModelListWithSearch = computed(() => { }) const modelCheckHandler = async (item: any) => { - console.log(item) - /** const response = await modelApi.check(item) const reader = response.body.getReader() const decoder = new TextDecoder() @@ -112,7 +110,6 @@ const modelCheckHandler = async (item: any) => { currentRef?.showErrorMask(checkMsg) } }) - */ } const duplicateName = async (item: any) => { const res = await modelApi.queryAll()