14 changed files with 3 additions and 825 deletions
@ -1,118 +0,0 @@ |
|||
import argparse |
|||
import logging |
|||
import sys |
|||
import warnings |
|||
|
|||
from deepsearcher.configuration import Configuration, init_config |
|||
from deepsearcher.offline_loading import load_from_local_files, load_from_website |
|||
from deepsearcher.online_query import query |
|||
from deepsearcher.utils import log |
|||
|
|||
httpx_logger = logging.getLogger("httpx") # disable openai's logger output |
|||
httpx_logger.setLevel(logging.WARNING) |
|||
|
|||
|
|||
warnings.simplefilter(action="ignore", category=FutureWarning) # disable warning output |
|||
|
|||
|
|||
def main(): |
|||
""" |
|||
Main entry point for the DeepSearcher CLI. |
|||
|
|||
This function parses command line arguments and executes the appropriate action |
|||
based on the subcommand provided (query or load). It handles the deprecated |
|||
command line format and provides helpful error messages. |
|||
|
|||
Returns: |
|||
None |
|||
""" |
|||
if "--query" in sys.argv or "--load" in sys.argv: |
|||
print("\033[91m[Deprecated]\033[0m The use of '--query' and '--load' is deprecated.") |
|||
print("Please use:") |
|||
print(" deepsearcher query <your_query> --max_iter 3") |
|||
print( |
|||
" deepsearcher load <your_local_path_or_url> --collection_name <your_collection_name> --collection_desc <your_collection_description>" |
|||
) |
|||
sys.exit(1) |
|||
|
|||
config = Configuration() # Customize your config here |
|||
init_config(config=config) |
|||
|
|||
parser = argparse.ArgumentParser(prog="deepsearcher", description="Deep Searcher.") |
|||
subparsers = parser.add_subparsers(dest="subcommand", title="subcommands") |
|||
|
|||
## Arguments of query |
|||
query_parser = subparsers.add_parser("query", help="Query a question or search topic.") |
|||
query_parser.add_argument("query", type=str, default="", help="query question or search topic.") |
|||
query_parser.add_argument( |
|||
"--max_iter", |
|||
type=int, |
|||
default=3, |
|||
help="Max iterations of reflection. Default is 3.", |
|||
) |
|||
|
|||
## Arguments of loading |
|||
load_parser = subparsers.add_parser( |
|||
"load", help="Load knowledge from local files or from URLs." |
|||
) |
|||
load_parser.add_argument( |
|||
"load_path", |
|||
type=str, |
|||
nargs="+", # 1 or more files or urls |
|||
help="Load knowledge from local files or from URLs.", |
|||
) |
|||
load_parser.add_argument( |
|||
"--batch_size", |
|||
type=int, |
|||
default=256, |
|||
help="Batch size for loading knowledge.", |
|||
) |
|||
load_parser.add_argument( |
|||
"--collection_name", |
|||
type=str, |
|||
default=None, |
|||
help="Destination collection name of loaded knowledge.", |
|||
) |
|||
load_parser.add_argument( |
|||
"--collection_desc", |
|||
type=str, |
|||
default=None, |
|||
help="Description of the collection.", |
|||
) |
|||
load_parser.add_argument( |
|||
"--force_new_collection", |
|||
type=bool, |
|||
default=False, |
|||
help="If you want to drop origin collection and create a new collection on every load, set to True", |
|||
) |
|||
|
|||
args = parser.parse_args() |
|||
if args.subcommand == "query": |
|||
final_answer, refs, consumed_tokens = query(args.query, max_iter=args.max_iter) |
|||
log.color_print("\n==== FINAL ANSWER====\n") |
|||
log.color_print(final_answer) |
|||
log.color_print("\n### References\n") |
|||
for i, ref in enumerate(refs): |
|||
log.color_print(f"{i + 1}. {ref.text[:60]}… {ref.reference}") |
|||
elif args.subcommand == "load": |
|||
urls = [url for url in args.load_path if url.startswith("http")] |
|||
local_files = [file for file in args.load_path if not file.startswith("http")] |
|||
kwargs = {} |
|||
if args.collection_name: |
|||
kwargs["collection_name"] = args.collection_name |
|||
if args.collection_desc: |
|||
kwargs["collection_description"] = args.collection_desc |
|||
if args.force_new_collection: |
|||
kwargs["force_new_collection"] = args.force_new_collection |
|||
if args.batch_size: |
|||
kwargs["batch_size"] = args.batch_size |
|||
if len(urls) > 0: |
|||
load_from_website(urls, **kwargs) |
|||
if len(local_files) > 0: |
|||
load_from_local_files(local_files, **kwargs) |
|||
else: |
|||
print("Please provide a query or a load argument.") |
|||
|
|||
|
|||
if __name__ == "__main__": |
|||
main() |
|||
@ -1,11 +0,0 @@ |
|||
from deepsearcher.loader.web_crawler.crawl4ai_crawler import Crawl4AICrawler |
|||
from deepsearcher.loader.web_crawler.docling_crawler import DoclingCrawler |
|||
from deepsearcher.loader.web_crawler.firecrawl_crawler import FireCrawlCrawler |
|||
from deepsearcher.loader.web_crawler.jina_crawler import JinaCrawler |
|||
|
|||
__all__ = [ |
|||
"FireCrawlCrawler", |
|||
"JinaCrawler", |
|||
"Crawl4AICrawler", |
|||
"DoclingCrawler", |
|||
] |
|||
@ -1,55 +0,0 @@ |
|||
from abc import ABC |
|||
from typing import List |
|||
|
|||
from langchain_core.documents import Document |
|||
|
|||
|
|||
class BaseCrawler(ABC): |
|||
""" |
|||
Abstract base class for web crawlers. |
|||
|
|||
This class defines the interface for crawling web pages and converting them |
|||
into Document objects for further processing. |
|||
""" |
|||
|
|||
def __init__(self, **kwargs): |
|||
""" |
|||
Initialize the crawler with optional keyword arguments. |
|||
|
|||
Args: |
|||
**kwargs: Optional keyword arguments for specific crawler implementations. |
|||
""" |
|||
pass |
|||
|
|||
def crawl_url(self, url: str, **crawl_kwargs) -> List[Document]: |
|||
""" |
|||
Crawl a single URL and convert it to Document objects. |
|||
|
|||
Args: |
|||
url: The URL to crawl. |
|||
**crawl_kwargs: Optional keyword arguments for the crawling process. |
|||
|
|||
Returns: |
|||
A list of Document objects containing the content and metadata from the URL. |
|||
|
|||
Note: |
|||
Implementations should include the URL reference in the metadata. |
|||
e.g. return [Document(page_content=..., metadata={"reference": "www.abc.com/page1.html"})] |
|||
""" |
|||
pass |
|||
|
|||
def crawl_urls(self, urls: List[str], **crawl_kwargs) -> List[Document]: |
|||
""" |
|||
Crawl multiple URLs and return a list of Document objects. |
|||
|
|||
Args: |
|||
urls: A list of URLs to crawl. |
|||
**crawl_kwargs: Optional keyword arguments for the crawling process. |
|||
|
|||
Returns: |
|||
A list of Document objects containing the content and metadata from all URLs. |
|||
""" |
|||
documents = [] |
|||
for url in urls: |
|||
documents.extend(self.crawl_url(url, **crawl_kwargs)) |
|||
return documents |
|||
@ -1,140 +0,0 @@ |
|||
import asyncio |
|||
from typing import List |
|||
|
|||
from langchain_core.documents import Document |
|||
|
|||
from deepsearcher.loader.web_crawler.base import BaseCrawler |
|||
from deepsearcher.utils import log |
|||
|
|||
|
|||
class Crawl4AICrawler(BaseCrawler): |
|||
""" |
|||
Web crawler using the Crawl4AI library. |
|||
|
|||
This crawler uses the Crawl4AI library to crawl web pages asynchronously and convert them |
|||
into markdown format for further processing. It supports both single-page crawling |
|||
and batch crawling of multiple pages. |
|||
""" |
|||
|
|||
def __init__(self, **kwargs): |
|||
""" |
|||
Initialize the Crawl4AICrawler. |
|||
|
|||
Args: |
|||
**kwargs: Optional keyword arguments. |
|||
browser_config: Configuration for the browser used by Crawl4AI. |
|||
""" |
|||
super().__init__(**kwargs) |
|||
self.crawler = None # Lazy init |
|||
self.browser_config = kwargs.get("browser_config", None) |
|||
|
|||
def _lazy_init(self): |
|||
""" |
|||
Initialize the crawler lazily when needed. |
|||
|
|||
This method creates the AsyncWebCrawler instance with the provided browser configuration |
|||
only when it's first needed, to avoid unnecessary initialization. |
|||
""" |
|||
from crawl4ai import AsyncWebCrawler, BrowserConfig |
|||
|
|||
if self.crawler is None: |
|||
config = BrowserConfig.from_kwargs(self.browser_config) if self.browser_config else None |
|||
self.crawler = AsyncWebCrawler(config=config) |
|||
|
|||
async def _async_crawl(self, url: str) -> Document: |
|||
""" |
|||
Asynchronously crawl a single URL. |
|||
|
|||
Args: |
|||
url: The URL to crawl. |
|||
|
|||
Returns: |
|||
A Document object with the markdown content and metadata from the URL. |
|||
""" |
|||
if self.crawler is None: |
|||
self._lazy_init() |
|||
|
|||
async with self.crawler as crawler: |
|||
result = await crawler.arun(url) |
|||
|
|||
markdown_content = result.markdown or "" |
|||
|
|||
metadata = { |
|||
"reference": url, |
|||
"success": result.success, |
|||
"status_code": result.status_code, |
|||
"media": result.media, |
|||
"links": result.links, |
|||
} |
|||
|
|||
if hasattr(result, "metadata") and result.metadata: |
|||
metadata["title"] = result.metadata.get("title", "") |
|||
metadata["author"] = result.metadata.get("author", "") |
|||
|
|||
return Document(page_content=markdown_content, metadata=metadata) |
|||
|
|||
def crawl_url(self, url: str) -> List[Document]: |
|||
""" |
|||
Crawl a single URL. |
|||
|
|||
Args: |
|||
url: The URL to crawl. |
|||
|
|||
Returns: |
|||
A list containing a single Document object with the markdown content and metadata, |
|||
or an empty list if an error occurs. |
|||
""" |
|||
try: |
|||
document = asyncio.run(self._async_crawl(url)) |
|||
return [document] |
|||
except Exception as e: |
|||
log.error(f"Error during crawling {url}: {e}") |
|||
return [] |
|||
|
|||
async def _async_crawl_many(self, urls: List[str]) -> List[Document]: |
|||
""" |
|||
Asynchronously crawl multiple URLs. |
|||
|
|||
Args: |
|||
urls: A list of URLs to crawl. |
|||
|
|||
Returns: |
|||
A list of Document objects with the markdown content and metadata from all URLs. |
|||
""" |
|||
if self.crawler is None: |
|||
self._lazy_init() |
|||
async with self.crawler as crawler: |
|||
results = await crawler.arun_many(urls) |
|||
documents = [] |
|||
for result in results: |
|||
markdown_content = result.markdown or "" |
|||
metadata = { |
|||
"reference": result.url, |
|||
"success": result.success, |
|||
"status_code": result.status_code, |
|||
"media": result.media, |
|||
"links": result.links, |
|||
} |
|||
if hasattr(result, "metadata") and result.metadata: |
|||
metadata["title"] = result.metadata.get("title", "") |
|||
metadata["author"] = result.metadata.get("author", "") |
|||
documents.append(Document(page_content=markdown_content, metadata=metadata)) |
|||
return documents |
|||
|
|||
def crawl_urls(self, urls: List[str], **crawl_kwargs) -> List[Document]: |
|||
""" |
|||
Crawl multiple URLs. |
|||
|
|||
Args: |
|||
urls: A list of URLs to crawl. |
|||
**crawl_kwargs: Optional keyword arguments for the crawling process. |
|||
|
|||
Returns: |
|||
A list of Document objects with the markdown content and metadata from all URLs, |
|||
or an empty list if an error occurs. |
|||
""" |
|||
try: |
|||
return asyncio.run(self._async_crawl_many(urls)) |
|||
except Exception as e: |
|||
log.error(f"Error during crawling {urls}: {e}") |
|||
return [] |
|||
@ -1,98 +0,0 @@ |
|||
from typing import List |
|||
|
|||
from langchain_core.documents import Document |
|||
|
|||
from deepsearcher.loader.web_crawler.base import BaseCrawler |
|||
from deepsearcher.utils import log |
|||
|
|||
|
|||
class DoclingCrawler(BaseCrawler): |
|||
""" |
|||
Web crawler using Docling's DocumentConverter and HierarchicalChunker. |
|||
|
|||
This crawler leverages Docling's capabilities to convert web pages into structured |
|||
documents and chunk them appropriately for further processing. |
|||
""" |
|||
|
|||
def __init__(self, **kwargs): |
|||
""" |
|||
Initialize the DoclingCrawler with DocumentConverter and HierarchicalChunker instances. |
|||
|
|||
Args: |
|||
**kwargs: Optional keyword arguments. |
|||
""" |
|||
super().__init__(**kwargs) |
|||
from docling.document_converter import DocumentConverter |
|||
from docling_core.transforms.chunker import HierarchicalChunker |
|||
|
|||
self.converter = DocumentConverter() |
|||
self.chunker = HierarchicalChunker() |
|||
|
|||
def crawl_url(self, url: str, **crawl_kwargs) -> List[Document]: |
|||
""" |
|||
Crawl a single URL using Docling's conversion and perform hierarchical chunking. |
|||
|
|||
Args: |
|||
url: The URL to crawl. |
|||
**crawl_kwargs: Optional keyword arguments for the crawling process. |
|||
|
|||
Returns: |
|||
A list of Document objects, each representing a chunk from the crawled URL. |
|||
|
|||
Raises: |
|||
IOError: If there is an error processing the URL. |
|||
""" |
|||
try: |
|||
# Use Docling to convert the URL to a document |
|||
conversion_result = self.converter.convert(url) |
|||
docling_document = conversion_result.document |
|||
|
|||
# Chunk the document using hierarchical chunking |
|||
chunks = list(self.chunker.chunk(docling_document)) |
|||
|
|||
documents = [] |
|||
for chunk in chunks: |
|||
metadata = {"reference": url, "text": chunk.text} |
|||
documents.append(Document(page_content=chunk.text, metadata=metadata)) |
|||
|
|||
return documents |
|||
|
|||
except Exception as e: |
|||
log.color_print(f"Error processing URL {url}: {str(e)}") |
|||
raise IOError(f"Failed to process URL {url}: {str(e)}") |
|||
|
|||
@property |
|||
def supported_file_types(self) -> List[str]: |
|||
""" |
|||
Return the list of file types and formats supported by Docling. |
|||
|
|||
Supported formats (refer to the official Docling documentation: https://docling-project.github.io/docling/usage/supported_formats/): |
|||
- PDF |
|||
- Office formats: DOCX, XLSX, PPTX |
|||
- Markdown |
|||
- AsciiDoc |
|||
- HTML, XHTML |
|||
- CSV |
|||
- Images: PNG, JPEG, TIFF, BMP |
|||
|
|||
Returns: |
|||
A list of file extensions supported by this crawler. |
|||
""" |
|||
return [ |
|||
"pdf", |
|||
"docx", |
|||
"xlsx", |
|||
"pptx", |
|||
"md", |
|||
"adoc", |
|||
"asciidoc", |
|||
"html", |
|||
"xhtml", |
|||
"csv", |
|||
"png", |
|||
"jpg", |
|||
"jpeg", |
|||
"tif", |
|||
"tiff", |
|||
"bmp", |
|||
] |
|||
@ -1,88 +0,0 @@ |
|||
import os |
|||
from typing import List, Optional |
|||
|
|||
from firecrawl import FirecrawlApp, ScrapeOptions |
|||
from langchain_core.documents import Document |
|||
|
|||
from deepsearcher.loader.web_crawler.base import BaseCrawler |
|||
|
|||
|
|||
class FireCrawlCrawler(BaseCrawler): |
|||
""" |
|||
Web crawler using the FireCrawl service. |
|||
|
|||
This crawler uses the FireCrawl service to crawl web pages and convert them |
|||
into markdown format for further processing. It supports both single-page scraping |
|||
and recursive crawling of multiple pages. |
|||
""" |
|||
|
|||
def __init__(self, **kwargs): |
|||
""" |
|||
Initialize the FireCrawlCrawler. |
|||
|
|||
Args: |
|||
**kwargs: Optional keyword arguments. |
|||
""" |
|||
super().__init__(**kwargs) |
|||
self.app = None |
|||
|
|||
def crawl_url( |
|||
self, |
|||
url: str, |
|||
max_depth: Optional[int] = None, |
|||
limit: Optional[int] = None, |
|||
allow_backward_links: Optional[bool] = None, |
|||
) -> List[Document]: |
|||
""" |
|||
Dynamically crawls a URL using either scrape_url or crawl_url: |
|||
|
|||
- Uses scrape_url for single-page extraction if no params are provided. |
|||
- Uses crawl_url to recursively gather pages when any param is provided. |
|||
|
|||
Args: |
|||
url (str): The starting URL to crawl. |
|||
max_depth (Optional[int]): Maximum depth for recursive crawling (default: 2). |
|||
limit (Optional[int]): Maximum number of pages to crawl (default: 20). |
|||
allow_backward_links (Optional[bool]): Allow crawling pages outside the URL's children (default: False). |
|||
|
|||
Returns: |
|||
List[Document]: List of Document objects with page content and metadata. |
|||
""" |
|||
# Lazy init |
|||
self.app = FirecrawlApp(api_key=os.getenv("FIRECRAWL_API_KEY")) |
|||
|
|||
# if user just inputs a single url as param |
|||
# scrape single page |
|||
if max_depth is None and limit is None and allow_backward_links is None: |
|||
# Call the new Firecrawl API, passing formats directly |
|||
scrape_response = self.app.scrape_url(url=url, formats=["markdown"]) |
|||
data = scrape_response.model_dump() |
|||
return [ |
|||
Document( |
|||
page_content=data.get("markdown", ""), |
|||
metadata={"reference": url, **data.get("metadata", {})}, |
|||
) |
|||
] |
|||
|
|||
# else, crawl multiple pages based on users' input params |
|||
# set default values if not provided |
|||
crawl_response = self.app.crawl_url( |
|||
url=url, |
|||
limit=limit or 20, |
|||
max_depth=max_depth or 2, |
|||
allow_backward_links=allow_backward_links or False, |
|||
scrape_options=ScrapeOptions(formats=["markdown"]), |
|||
poll_interval=5, |
|||
) |
|||
items = crawl_response.model_dump().get("data", []) |
|||
|
|||
documents: List[Document] = [] |
|||
for item in items: |
|||
# Support items that are either dicts or Pydantic sub-models |
|||
item_dict = item.model_dump() if hasattr(item, "model_dump") else item |
|||
md = item_dict.get("markdown", "") |
|||
meta = item_dict.get("metadata", {}) |
|||
meta["reference"] = meta.get("url", url) |
|||
documents.append(Document(page_content=md, metadata=meta)) |
|||
|
|||
return documents |
|||
@ -1,62 +0,0 @@ |
|||
import os |
|||
from typing import List |
|||
|
|||
import requests |
|||
from langchain_core.documents import Document |
|||
|
|||
from deepsearcher.loader.web_crawler.base import BaseCrawler |
|||
|
|||
|
|||
class JinaCrawler(BaseCrawler): |
|||
""" |
|||
Web crawler using Jina AI's rendering service. |
|||
|
|||
This crawler uses Jina AI's rendering service to crawl web pages and convert them |
|||
into markdown format for further processing. |
|||
""" |
|||
|
|||
def __init__(self, **kwargs): |
|||
""" |
|||
Initialize the JinaCrawler. |
|||
|
|||
Args: |
|||
**kwargs: Optional keyword arguments. |
|||
|
|||
Raises: |
|||
ValueError: If the JINA_API_TOKEN environment variable is not set. |
|||
""" |
|||
super().__init__(**kwargs) |
|||
self.jina_api_token = os.getenv("JINA_API_TOKEN") or os.getenv("JINAAI_API_KEY") |
|||
if not self.jina_api_token: |
|||
raise ValueError("Missing JINA_API_TOKEN environment variable") |
|||
|
|||
def crawl_url(self, url: str) -> List[Document]: |
|||
""" |
|||
Crawl a single URL using Jina AI's rendering service. |
|||
|
|||
Args: |
|||
url: The URL to crawl. |
|||
|
|||
Returns: |
|||
A list containing a single Document object with the markdown content and metadata. |
|||
|
|||
Raises: |
|||
HTTPError: If the request to Jina AI's service fails. |
|||
""" |
|||
jina_url = f"https://r.jina.ai/{url}" |
|||
headers = { |
|||
"Authorization": f"Bearer {self.jina_api_token}", |
|||
"X-Return-Format": "markdown", |
|||
} |
|||
|
|||
response = requests.get(jina_url, headers=headers) |
|||
response.raise_for_status() |
|||
|
|||
markdown_content = response.text |
|||
metadata = { |
|||
"reference": url, |
|||
"status_code": response.status_code, |
|||
"headers": dict(response.headers), |
|||
} |
|||
|
|||
return [Document(page_content=markdown_content, metadata=metadata)] |
|||
@ -1,40 +0,0 @@ |
|||
#!/usr/bin/env python3 |
|||
""" |
|||
只测试网页搜索功能 |
|||
""" |
|||
|
|||
import sys |
|||
import os |
|||
sys.path.append(os.path.dirname(os.path.abspath(__file__))) |
|||
|
|||
from deepsearcher.web_search import WebSearch |
|||
|
|||
def test_web_search(): |
|||
"""测试网页搜索功能""" |
|||
print("=== 测试网页搜索功能 ===") |
|||
|
|||
# 初始化网页搜索 |
|||
web_search = WebSearch() |
|||
|
|||
# 测试查询 |
|||
test_query = "Milvus是什么" |
|||
print(f"测试查询: {test_query}") |
|||
|
|||
# 执行搜索 |
|||
results = web_search.search_with_retry(test_query, size=4) |
|||
|
|||
if results: |
|||
print(f"✅ 成功找到 {len(results)} 个搜索结果:") |
|||
for i, result in enumerate(results, 1): |
|||
print(f"\n--- 结果 {i} ---") |
|||
print(f"标题: {result.metadata.get('title', 'N/A')}") |
|||
print(f"链接: {result.reference}") |
|||
print(f"分数: {result.score}") |
|||
print(f"内容长度: {len(result.text)} 字符") |
|||
print(f"内容预览: {result.text[:200]}...") |
|||
print(f"来源: {result.metadata.get('source', 'N/A')}") |
|||
else: |
|||
print("❌ 未找到搜索结果") |
|||
|
|||
if __name__ == "__main__": |
|||
test_web_search() |
|||
@ -1,75 +0,0 @@ |
|||
#!/usr/bin/env python3 |
|||
""" |
|||
测试网页搜索功能 |
|||
""" |
|||
|
|||
import sys |
|||
import os |
|||
sys.path.append(os.path.dirname(os.path.abspath(__file__))) |
|||
|
|||
from deepsearcher.web_search import WebSearch |
|||
from deepsearcher import configuration |
|||
|
|||
def test_web_search(): |
|||
"""测试网页搜索功能""" |
|||
print("=== 测试网页搜索功能 ===") |
|||
|
|||
# 初始化网页搜索 |
|||
web_search = WebSearch() |
|||
|
|||
# 测试查询 |
|||
test_query = "Milvus是什么" |
|||
print(f"测试查询: {test_query}") |
|||
|
|||
# 执行搜索 |
|||
results = web_search.search_with_retry(test_query, size=4) |
|||
|
|||
if results: |
|||
print(f"找到 {len(results)} 个搜索结果:") |
|||
for i, result in enumerate(results, 1): |
|||
print(f"\n--- 结果 {i} ---") |
|||
print(f"标题: {result.metadata.get('title', 'N/A')}") |
|||
print(f"链接: {result.reference}") |
|||
print(f"分数: {result.score}") |
|||
print(f"内容长度: {len(result.text)} 字符") |
|||
print(f"内容预览: {result.text[:200]}...") |
|||
else: |
|||
print("未找到搜索结果") |
|||
|
|||
def test_integration(): |
|||
"""测试与DeepSearch的集成""" |
|||
print("\n=== 测试与DeepSearch的集成 ===") |
|||
|
|||
# 初始化配置 |
|||
configuration.init_config(configuration.config) |
|||
|
|||
# 创建DeepSearch实例（启用网页搜索） |
|||
from deepsearcher.agent.deep_search import DeepSearch |
|||
|
|||
searcher = DeepSearch( |
|||
llm=configuration.llm, |
|||
embedding_model=configuration.embedding_model, |
|||
vector_db=configuration.vector_db, |
|||
max_iter=2, |
|||
enable_web_search=True |
|||
) |
|||
|
|||
# 测试查询 |
|||
test_query = "Milvus是什么" |
|||
print(f"测试查询: {test_query}") |
|||
|
|||
# 执行搜索 |
|||
results, sub_queries = searcher.retrieve(test_query, max_iter=2) |
|||
|
|||
print(f"生成的子问题: {sub_queries}") |
|||
print(f"找到 {len(results)} 个搜索结果") |
|||
# 显示结果统计 |
|||
web_results = [r for r in results if r.metadata and r.metadata.get("source") == "webpage"] |
|||
vector_results = [r for r in results if not r.metadata or r.metadata.get("source") != "webpage"] |
|||
|
|||
print(f"网页搜索结果: {len(web_results)} 个") |
|||
print(f"向量数据库结果: {len(vector_results)} 个") |
|||
|
|||
if __name__ == "__main__": |
|||
test_web_search() |
|||
test_integration() |
|||
Loading…
Reference in new issue