mirror of
https://gh-proxy.org/https://github.com/klinsmaya/SQLBot.git
synced 2026-11-08 01:06:59 +08:00
perf: Optimize VLLM OpenAI integration vLLM model validation result parsing
This commit is contained in:
@@ -66,13 +66,14 @@ class OpenAIvLLM(BaseLLM):
|
||||
openai_api_key=self.config.api_key or 'Empty',
|
||||
openai_api_base=self.config.api_base_url,
|
||||
model_name=self.config.model_name,
|
||||
streaming=True,
|
||||
**self.config.additional_params,
|
||||
)
|
||||
class OpenAILLM(BaseLLM):
|
||||
def _init_llm(self) -> BaseChatModel:
|
||||
return BaseChatOpenAI(
|
||||
model=self.config.model_name,
|
||||
api_key=self.config.api_key,
|
||||
api_key=self.config.api_key or 'Empty',
|
||||
base_url=self.config.api_base_url,
|
||||
stream_usage=True,
|
||||
**self.config.additional_params,
|
||||
|
||||
@@ -28,7 +28,10 @@ async def check_llm(info: AiModelCreator, trans: Trans):
|
||||
)
|
||||
llm_instance = LLMFactory.create_llm(config)
|
||||
async for chunk in llm_instance.llm.astream("1+1=?"):
|
||||
if chunk and chunk.content:
|
||||
SQLBotLogUtil.info(chunk)
|
||||
if chunk and isinstance(chunk, str):
|
||||
yield json.dumps({"content": chunk}) + "\n"
|
||||
if chunk and isinstance(chunk, dict) and chunk.content:
|
||||
yield json.dumps({"content": chunk.content}) + "\n"
|
||||
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user