Add filetype key for web chunk

2025-03-15 15:35:09 +01:00 · 2025-03-15 15:35:09 +01:00 · 37aa171924
commit 37aa171924
parent 865c2c3b8b
1 changed files with 1 additions and 0 deletions
--- a/generic_rag/parsers/parser.py
+++ b/generic_rag/parsers/parser.py
@ -52,6 +52,7 @@ def add_urls(vector_store: Chroma, urls: list[str], chunk_size: int) -> None:

        for split in splits:
            split.metadata["source"] = url
+            split.metadata["filetype"] = "web"

        all_splits.extend(splits)