refac: RAG_WEB_LOADER -> RAG_WEB_LOADER_ENGINE

rgaricano · Feb 18, 2025 · bc82f48 · bc82f48
1 parent 66c2acc
commit bc82f48
Show file tree

Hide file tree

Showing 6 changed files with 16 additions and 16 deletions.
diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py
@@ -1853,10 +1853,10 @@ class BannerModel(BaseModel):
     int(os.getenv("RAG_WEB_SEARCH_CONCURRENT_REQUESTS", "10")),
 )
 
-RAG_WEB_LOADER = PersistentConfig(
-    "RAG_WEB_LOADER",
-    "rag.web.loader",
-    os.environ.get("RAG_WEB_LOADER", "safe_web")
+RAG_WEB_LOADER_ENGINE = PersistentConfig(
+    "RAG_WEB_LOADER_ENGINE",
+    "rag.web.loader.engine",
+    os.environ.get("RAG_WEB_LOADER_ENGINE", "safe_web")
 )
 
 RAG_WEB_SEARCH_TRUST_ENV = PersistentConfig(
@@ -1867,7 +1867,7 @@ class BannerModel(BaseModel):
 
 PLAYWRIGHT_WS_URI = PersistentConfig(
     "PLAYWRIGHT_WS_URI",
-    "rag.web.loader.playwright.ws.uri",
+    "rag.web.loader.engine.playwright.ws.uri",
     os.environ.get("PLAYWRIGHT_WS_URI", None)
 )
 

diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py
@@ -140,7 +140,7 @@
     AUDIO_TTS_AZURE_SPEECH_REGION,
     AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT,
     PLAYWRIGHT_WS_URI,
-    RAG_WEB_LOADER,
+    RAG_WEB_LOADER_ENGINE,
     WHISPER_MODEL,
     DEEPGRAM_API_KEY,
     WHISPER_MODEL_AUTO_UPDATE,
@@ -561,7 +561,7 @@ async def lifespan(app: FastAPI):
 
 app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT
 app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS
-app.state.config.RAG_WEB_LOADER = RAG_WEB_LOADER
+app.state.config.RAG_WEB_LOADER_ENGINE = RAG_WEB_LOADER_ENGINE
 app.state.config.RAG_WEB_SEARCH_TRUST_ENV = RAG_WEB_SEARCH_TRUST_ENV
 app.state.config.PLAYWRIGHT_WS_URI = PLAYWRIGHT_WS_URI
 

diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py
@@ -25,7 +25,7 @@
 )
 from langchain_core.documents import Document
 from open_webui.constants import ERROR_MESSAGES
-from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH, PLAYWRIGHT_WS_URI, RAG_WEB_LOADER
+from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH, PLAYWRIGHT_WS_URI, RAG_WEB_LOADER_ENGINE
 from open_webui.env import SRC_LOG_LEVELS
 
 log = logging.getLogger(__name__)
@@ -352,9 +352,9 @@ async def aload(self) -> list[Document]:
         """Load data into Document objects."""
         return [document async for document in self.alazy_load()]
 
-RAG_WEB_LOADERS = defaultdict(lambda: SafeWebBaseLoader)
-RAG_WEB_LOADERS["playwright"] = SafePlaywrightURLLoader
-RAG_WEB_LOADERS["safe_web"] = SafeWebBaseLoader
+RAG_WEB_LOADER_ENGINES = defaultdict(lambda: SafeWebBaseLoader)
+RAG_WEB_LOADER_ENGINES["playwright"] = SafePlaywrightURLLoader
+RAG_WEB_LOADER_ENGINES["safe_web"] = SafeWebBaseLoader
 
 def get_web_loader(
     urls: Union[str, Sequence[str]],
@@ -377,9 +377,9 @@ def get_web_loader(
         web_loader_args["playwright_ws_url"] = PLAYWRIGHT_WS_URI.value
 
     # Create the appropriate WebLoader based on the configuration
-    WebLoaderClass = RAG_WEB_LOADERS[RAG_WEB_LOADER.value]
+    WebLoaderClass = RAG_WEB_LOADER_ENGINES[RAG_WEB_LOADER_ENGINE.value]
     web_loader = WebLoaderClass(**web_loader_args)
 
-    log.debug("Using RAG_WEB_LOADER %s for %s URLs", web_loader.__class__.__name__, len(safe_urls))
+    log.debug("Using RAG_WEB_LOADER_ENGINE %s for %s URLs", web_loader.__class__.__name__, len(safe_urls))
 
     return web_loader
diff --git a/backend/start.sh b/backend/start.sh
@@ -4,7 +4,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 cd "$SCRIPT_DIR" || exit
 
 # Add conditional Playwright browser installation
-if [[ "${RAG_WEB_LOADER,,}" == "playwright" ]]; then
+if [[ "${RAG_WEB_LOADER_ENGINE,,}" == "playwright" ]]; then
     if [[ -z "${PLAYWRIGHT_WS_URI}" ]]; then
         echo "Installing Playwright browsers..."
         playwright install chromium

diff --git a/backend/start_windows.bat b/backend/start_windows.bat
@@ -7,7 +7,7 @@ SET "SCRIPT_DIR=%~dp0"
 cd /d "%SCRIPT_DIR%" || exit /b
 
 :: Add conditional Playwright browser installation
-IF /I "%RAG_WEB_LOADER%" == "playwright" (
+IF /I "%RAG_WEB_LOADER_ENGINE%" == "playwright" (
     IF "%PLAYWRIGHT_WS_URI%" == "" (
         echo Installing Playwright browsers...
         playwright install chromium

diff --git a/docker-compose.playwright.yaml b/docker-compose.playwright.yaml
@@ -6,5 +6,5 @@ services:
 
   open-webui:
     environment:
-      - 'RAG_WEB_LOADER=playwright'
+      - 'RAG_WEB_LOADER_ENGINE=playwright'
       - 'PLAYWRIGHT_WS_URI=ws://playwright:3000'