|
|
@ -10,10 +10,10 @@ import os |
|
|
|
import asyncio |
|
|
|
import json |
|
|
|
import queue |
|
|
|
from typing import AsyncGenerator |
|
|
|
from collections.abc import AsyncGenerator |
|
|
|
|
|
|
|
from deepsearcher.configuration import Configuration, init_config |
|
|
|
from deepsearcher.offline_loading import load_from_local_files, load_from_website |
|
|
|
from deepsearcher.offline_loading import load_from_local_files |
|
|
|
from deepsearcher.online_query import query |
|
|
|
from deepsearcher.utils.message_stream import get_message_stream |
|
|
|
|
|
|
@ -146,63 +146,6 @@ def load_files( |
|
|
|
raise HTTPException(status_code=500, detail=str(e)) |
|
|
|
|
|
|
|
|
|
|
|
@app.post("/load-website/") |
|
|
|
def load_website( |
|
|
|
urls: str | list[str] = Body( |
|
|
|
..., |
|
|
|
description="A list of URLs of websites to be loaded.", |
|
|
|
examples=["https://milvus.io/docs/overview.md"], |
|
|
|
), |
|
|
|
collection_name: str = Body( |
|
|
|
None, |
|
|
|
description="Optional name for the collection.", |
|
|
|
examples=["my_collection"], |
|
|
|
), |
|
|
|
collection_description: str = Body( |
|
|
|
None, |
|
|
|
description="Optional description for the collection.", |
|
|
|
examples=["This is a test collection."], |
|
|
|
), |
|
|
|
batch_size: int = Body( |
|
|
|
None, |
|
|
|
description="Optional batch size for the collection.", |
|
|
|
examples=[256], |
|
|
|
), |
|
|
|
force_rebuild: bool = Body( |
|
|
|
False, |
|
|
|
description="Whether to force rebuild the collection if it already exists.", |
|
|
|
examples=[False], |
|
|
|
), |
|
|
|
): |
|
|
|
""" |
|
|
|
Load website content into the vector database. |
|
|
|
|
|
|
|
Args: |
|
|
|
urls (Union[str, List[str]]): URLs of websites to load. |
|
|
|
collection_name (str, optional): Name for the collection. Defaults to None. |
|
|
|
collection_description (str, optional): Description for the collection. Defaults to None. |
|
|
|
batch_size (int, optional): Batch size for processing. Defaults to None. |
|
|
|
force_rebuild (bool, optional): Whether to force rebuild the collection. Defaults to False. |
|
|
|
|
|
|
|
Returns: |
|
|
|
dict: A dictionary containing a success message. |
|
|
|
|
|
|
|
Raises: |
|
|
|
HTTPException: If loading website content fails. |
|
|
|
""" |
|
|
|
try: |
|
|
|
load_from_website( |
|
|
|
urls=urls, |
|
|
|
collection_name=collection_name, |
|
|
|
collection_description=collection_description, |
|
|
|
batch_size=batch_size if batch_size is not None else 8, |
|
|
|
force_rebuild=force_rebuild, |
|
|
|
) |
|
|
|
return {"message": "成功加载"} |
|
|
|
except Exception as e: |
|
|
|
raise HTTPException(status_code=500, detail=str(e)) |
|
|
|
|
|
|
|
|
|
|
|
@app.get("/query-stream/") |
|
|
|
async def perform_query_stream( |
|
|
|
original_query: str = Query( |
|
|
|