Skip to content

Commit

Permalink
update readme
Browse files Browse the repository at this point in the history
  • Loading branch information
@BoPeng
BoPeng committed Jan 30, 2025
1 parent 1217d42 commit 467c5aa
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 58 deletions.
8 changes: 8 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,14 @@ description = '''A new or used Go Pro version 11, 12 or 13 in
min_price = 100
max_price = 200

[item.name2]
keywords = 'something rare'
description = '''A rare item that has to be searched nationwide and be shipped.
listings from any location are acceptable.'''
search_region = 'usa'
delivery_method = 'shipping'
acceptable_locations = []

[user.user1]
pushbullet_token = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
```
Expand Down
11 changes: 7 additions & 4 deletions example_config.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,18 +38,21 @@ pushbullet_token = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
pushbullet_token = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'


# OPTIONAL a product
# search an item across the usa, acceptable listing from any location
# and perform one search each day
[item.name1]
keywords = 'search word one'
search_region = 'usa'
search_interval = '1d'
delivery_method = 'shipping'
acceptable_locations = []

# Complete item
# search for a local item with longer description
[item.name2]
# REUIRED string or list of strings
keywords = ['search word one', 'search word two']
# OPTIONAL, recommended for AI
description = "it should be from manufacture, the seller should not offer shipping."
# OPTIONAL string or list of strings, default to all marketplaces
marketplace = 'facebook'
# OPTIONAL words that should not be in the title
exclude_keywords = ['exclude word one', 'exclude word two']
# OPTIONAL product specific user to notify
Expand Down
4 changes: 3 additions & 1 deletion src/ai_marketplace_monitor/ai.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,9 @@ class OpenAIBackend(AIBackend):
def connect(self: "OpenAIBackend") -> None:
if self.client is None:
self.client = OpenAI(
api_key=self.config["api_key"], base_url=self.config.get("base_url", self.base_url)
api_key=self.config["api_key"],
base_url=self.config.get("base_url", self.base_url),
timeout=10,
)

def confirm(
Expand Down
117 changes: 64 additions & 53 deletions src/ai_marketplace_monitor/monitor.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import os
import random
import time
from logging import Logger
from typing import Any, ClassVar, Dict, List

import schedule
from playwright.sync_api import Browser, sync_playwright
from playwright.sync_api import Browser, Playwright, sync_playwright
from rich.pretty import pretty_repr

from .ai import AIBackend, DeepSeekBackend, OpenAIBackend
Expand Down Expand Up @@ -44,6 +43,7 @@ def __init__(
self.config_hash: str | None = None
self.headless = headless
self.ai_agents: List[AIBackend] = []
self.playwright: Playwright | None = None
self.logger = logger
if clear_cache:
cache.clear()
Expand Down Expand Up @@ -81,9 +81,9 @@ def load_ai_agents(self: "MarketplaceMonitor") -> None:
ai_class = supported_ai_backends[ai_name]
ai_class.validate(ai_config)
try:
self.logger.info(f"Connecting to {ai_name}")
self.ai_agents.append(ai_class(config=ai_config, logger=self.logger))
self.ai_agents[-1].connect()
self.logger.info(f"Connected to {ai_name}")
# if one works, do not try to load another one
break
except Exception as e:
Expand Down Expand Up @@ -130,62 +130,71 @@ def search_item(

def schedule_jobs(self: "MarketplaceMonitor") -> None:
"""Schedule jobs to run periodically."""
# start a browser with playwright
with sync_playwright() as p:
# Open a new browser page.
browser: Browser = p.chromium.launch(headless=self.headless)
# we reload the config file each time when a scan action is completed
# this allows users to add/remove products dynamically.
self.load_config_file()
self.load_ai_agents()
# start a browser with playwright, cannot use with statement since the jobs will be
# executed outside of the scope by schedule job runner
self.playwright = sync_playwright().start()
# Open a new browser page.
assert self.playwright is not None
browser: Browser = self.playwright.chromium.launch(headless=self.headless)
# we reload the config file each time when a scan action is completed
# this allows users to add/remove products dynamically.
self.load_config_file()
self.load_ai_agents()

assert self.config is not None
for marketplace_name, marketplace_config in self.config["marketplace"].items():
marketplace_class = supported_marketplaces[marketplace_name]
if marketplace_name in self.active_marketplaces:
marketplace = self.active_marketplaces[marketplace_name]
else:
marketplace = marketplace_class(marketplace_name, browser, self.logger)
self.active_marketplaces[marketplace_name] = marketplace

# Configure might have been changed
marketplace.configure(marketplace_config)

for item_name, item_config in self.config["item"].items():
if (
"marketplace" not in item_config
or item_config["marketplace"] == marketplace_name
):
if not item_config.get("enabled", True):
continue
# wait for some time before next search
# interval (in minutes) can be defined both for the marketplace
# if there is any configuration file change, stop sleeping and search again
search_interval = max(
item_config.get(
"search_interval", marketplace_config.get("search_interval", 30)
),
1,
)
max_search_interval = max(
item_config.get(marketplace_config.get("max_search_interval", 1)),
search_interval,
)
schedule.every(
random.randint(search_interval, max_search_interval)
).minutes.do(
self.search_item,
marketplace_name,
marketplace_config,
marketplace,
item_name,
item_config,
)
assert self.config is not None
for marketplace_name, marketplace_config in self.config["marketplace"].items():
marketplace_class = supported_marketplaces[marketplace_name]
if marketplace_name in self.active_marketplaces:
marketplace = self.active_marketplaces[marketplace_name]
else:
marketplace = marketplace_class(marketplace_name, browser, self.logger)
self.active_marketplaces[marketplace_name] = marketplace

# Configure might have been changed
marketplace.configure(marketplace_config)

for item_name, item_config in self.config["item"].items():
if (
"marketplace" not in item_config
or item_config["marketplace"] == marketplace_name
):
if not item_config.get("enabled", True):
continue
# wait for some time before next search
# interval (in minutes) can be defined both for the marketplace
# if there is any configuration file change, stop sleeping and search again
search_interval = max(
item_config.get(
"search_interval", marketplace_config.get("search_interval", 30)
),
1,
)
max_search_interval = max(
item_config.get(
"max_search_interval",
marketplace_config.get("max_search_interval", 1),
),
search_interval,
)
self.logger.info(
f"Scheduling to search for {item_name} every {search_interval} {"" if search_interval == max_search_interval else f'to {max_search_interval}'} minutes"
)
schedule.every(search_interval).to(max_search_interval).minutes.do(
self.search_item,
marketplace_name,
marketplace_config,
marketplace,
item_name,
item_config,
)

def start_monitor(self: "MarketplaceMonitor") -> None:
"""Main function to monitor the marketplace."""
while True:
self.schedule_jobs()
# run all jobs at the first time, then on their own
# schedule
schedule.run_all()
while True:
schedule.run_pending()
sleep_with_watchdog(
Expand All @@ -204,6 +213,8 @@ def stop_monitor(self: "MarketplaceMonitor") -> None:
"""Stop the monitor."""
for marketplace in self.active_marketplaces.values():
marketplace.stop()
if self.playwright is not None:
self.playwright.stop()
cache.close()

def check_items(
Expand Down

0 comments on commit 467c5aa

Please sign in to comment.