From 5e93efe9b217351d3aabd5fda864eae6f6a1d7f9 Mon Sep 17 00:00:00 2001 From: miro Date: Sun, 30 Jun 2024 22:18:36 +0100 Subject: [PATCH 01/12] feat!:configurable_database_backend allow several database backends Update test/unittests/test_db.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- README.md | 2 + hivemind_core/database.py | 670 +++++++++++++++++++++++++++----------- hivemind_core/protocol.py | 11 +- hivemind_core/scripts.py | 381 ++++++++++++++++------ hivemind_core/service.py | 80 +++-- setup.py | 2 +- test/unittests/test_db.py | 219 ++++++++++--- 7 files changed, 997 insertions(+), 368 deletions(-) diff --git a/README.md b/README.md index 0b18dfc..cf493da 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Work in progress documentation can be found in the [docs](https://jarbashivemind You can also join the [Hivemind Matrix chat](https://matrix.to/#/#jarbashivemind:matrix.org) for general news, support and chit chat + + # Usage ``` diff --git a/hivemind_core/database.py b/hivemind_core/database.py index 02e1ce0..2b283dd 100644 --- a/hivemind_core/database.py +++ b/hivemind_core/database.py @@ -1,254 +1,546 @@ +import abc import json -from functools import wraps +from dataclasses import dataclass, field from typing import List, Dict, Union, Any, Optional, Iterable -from json_database import JsonDatabaseXDG +from json_database import JsonStorageXDG from ovos_utils.log import LOG +try: + import redis +except ImportError: + redis = None -def cast_to_client_obj(): - valid_kwargs: Iterable[str] = ( - "client_id", - "api_key", - "name", - "description", - "is_admin", - "last_seen", - "blacklist", - "allowed_types", - "crypto_key", - "password", - "can_broadcast", - "can_escalate", - "can_propagate", - ) - - def _handler(func): - def _cast(ret): - if ret is None or isinstance(ret, Client): - return ret - if isinstance(ret, list): - return [_cast(r) for r in ret] - if isinstance(ret, dict): - if not all((k in valid_kwargs for k in ret.keys())): - raise RuntimeError(f"{func} returned a dict with unknown keys") - return Client(**ret) - - raise TypeError( - "cast_to_client_obj decorator can only be used in functions that return None, dict, Client or a list of those types" - ) - - @wraps(func) - def call_function(*args, **kwargs): - ret = func(*args, **kwargs) - return _cast(ret) - - return call_function - - return _handler +ClientDict = Dict[str, Union[str, int, float, List[str]]] +ClientTypes = Union[None, 'Client', + str, # json + ClientDict, # dict + List[Union[str, ClientDict, 'Client']] # list of dicts/json/Client + ] + +def cast2client(ret: ClientTypes) -> Optional[Union['Client', List['Client']]]: + """ + Convert different input types (str, dict, list) to Client instances. + + Args: + ret: The object to be cast, can be a string, dictionary, or list. + + Returns: + A single Client instance or a list of Clients if ret is a list. + """ + if ret is None or isinstance(ret, Client): + return ret + if isinstance(ret, str) or isinstance(ret, dict): + return Client.deserialize(ret) + if isinstance(ret, list): + return [cast2client(r) for r in ret] + raise TypeError("not a client object") + + +@dataclass class Client: - def __init__( - self, - client_id: int, - api_key: str, - name: str = "", - description: str = "", - is_admin: bool = False, - last_seen: float = -1, - blacklist: Optional[Dict[str, List[str]]] = None, - allowed_types: Optional[List[str]] = None, - crypto_key: Optional[str] = None, - password: Optional[str] = None, - can_broadcast: bool = True, - can_escalate: bool = True, - can_propagate: bool = True, - ): - self.client_id = client_id - self.description = description - self.api_key = api_key - self.name = name - self.last_seen = last_seen - self.is_admin = is_admin - self.crypto_key = crypto_key - self.password = password - self.blacklist = blacklist or {"messages": [], "skills": [], "intents": []} - self.allowed_types = allowed_types or ["recognizer_loop:utterance", - "recognizer_loop:record_begin", - "recognizer_loop:record_end", - "recognizer_loop:audio_output_start", - "recognizer_loop:audio_output_end", - 'recognizer_loop:b64_transcribe', - 'speak:b64_audio', - "ovos.common_play.SEI.get.response"] + client_id: int + api_key: str + name: str = "" + description: str = "" + is_admin: bool = False + last_seen: float = -1 + intent_blacklist: List[str] = field(default_factory=list) + skill_blacklist: List[str] = field(default_factory=list) + message_blacklist: List[str] = field(default_factory=list) + allowed_types: List[str] = field(default_factory=list) + crypto_key: Optional[str] = None + password: Optional[str] = None + can_broadcast: bool = True + can_escalate: bool = True + can_propagate: bool = True + + def __post_init__(self): + """ + Initializes the allowed types for the Client instance if not provided. + """ + self.allowed_types = self.allowed_types or ["recognizer_loop:utterance", + "recognizer_loop:record_begin", + "recognizer_loop:record_end", + "recognizer_loop:audio_output_start", + "recognizer_loop:audio_output_end", + 'recognizer_loop:b64_transcribe', + 'speak:b64_audio', + "ovos.common_play.SEI.get.response"] if "recognizer_loop:utterance" not in self.allowed_types: self.allowed_types.append("recognizer_loop:utterance") - self.can_broadcast = can_broadcast - self.can_escalate = can_escalate - self.can_propagate = can_propagate + + def serialize(self) -> str: + """ + Serializes the Client instance into a JSON string. + + Returns: + A JSON string representing the client data. + """ + return json.dumps(self.__dict__, sort_keys=True, ensure_ascii=False) + + @staticmethod + def deserialize(client_data: Union[str, Dict]) -> 'Client': + """ + Deserialize a client from JSON string or dictionary into a Client instance. + + Args: + client_data: The data to be deserialized, either a string or dictionary. + + Returns: + A Client instance. + """ + if isinstance(client_data, str): + client_data = json.loads(client_data) + # TODO filter kwargs with inspect + return Client(**client_data) def __getitem__(self, item: str) -> Any: - return self.__dict__.get(item) + """ + Access attributes of the client via item access. + + Args: + item: The name of the attribute. + + Returns: + The value of the attribute. + + Raises: + KeyError: If the attribute does not exist. + """ + if hasattr(self, item): + return getattr(self, item) + raise KeyError(f"Unknown key: {item}") def __setitem__(self, key: str, value: Any): + """ + Set attributes of the client via item access. + + Args: + key: The name of the attribute. + value: The value to set. + + Raises: + ValueError: If the attribute does not exist. + """ if hasattr(self, key): setattr(self, key, value) else: - raise ValueError("unknown property") + raise ValueError(f"Unknown property: {key}") - def __eq__(self, other: Union[object, dict]) -> bool: - if not isinstance(other, dict): - other = other.__dict__ - if self.__dict__ == other: - return True + def __eq__(self, other: Any) -> bool: + """ + Compares two Client instances for equality based on their serialized data. + + Args: + other: The other Client or Client-compatible object to compare with. + + Returns: + True if the clients are equal, False otherwise. + """ + try: + other = cast2client(other) + except: + pass + if isinstance(other, Client): + return self.serialize() == other.serialize() return False def __repr__(self) -> str: - return str(self.__dict__) + """ + Returns a string representation of the Client instance. + Returns: + A string representing the client. + """ + return self.serialize() -class ClientDatabase(JsonDatabaseXDG): - def __init__(self): - super().__init__("clients", subfolder="hivemind") - def update_timestamp(self, key: str, timestamp: float) -> bool: - user = self.get_client_by_api_key(key) - if user is None: - return False - item_id = self.get_item_id(user) - user["last_seen"] = timestamp - self.update_item(item_id, user) +class AbstractDB(abc.ABC): + """ + Abstract base class for all database implementations. + + All database implementations should derive from this class and implement + the abstract methods. + """ + + @abc.abstractmethod + def add_item(self, client: Client) -> bool: + """ + Add a client to the database. + + Args: + client: The client to be added. + + Returns: + True if the addition was successful, False otherwise. + """ + pass + + @abc.abstractmethod + def delete_item(self, client: Client) -> bool: + """ + Delete a client from the database. + + Args: + client: The client to be deleted. + + Returns: + True if the deletion was successful, False otherwise. + """ + pass + + def update_item(self, client: Client) -> bool: + """ + Update an existing client in the database. + + Args: + client: The client to be updated. + + Returns: + True if the update was successful, False otherwise. + """ + return self.add_item(client) + + def replace_item(self, old_client: Client, new_client: Client) -> bool: + """ + Replace an old client with a new client. + + Args: + old_client: The old client to be replaced. + new_client: The new client to add. + + Returns: + True if the replacement was successful, False otherwise. + """ + self.delete_item(old_client) + return self.add_item(new_client) + + @abc.abstractmethod + def search_by_value(self, key: str, val: Union[str, bool, int, float]) -> List[Client]: + """ + Search for clients by a specific key-value pair. + + Args: + key: The key to search by. + val: The value to search for. + + Returns: + A list of clients that match the search criteria. + """ + pass + + @abc.abstractmethod + def __len__(self) -> int: + """ + Get the number of items in the database. + + Returns: + The number of items in the database. + """ + return 0 + + @abc.abstractmethod + def __iter__(self) -> Iterable['Client']: + """ + Iterate over all clients in the database. + + Returns: + An iterator over the clients in the database. + """ + pass + + def commit(self) -> bool: + """ + Commit changes to the database. + + Returns: + True if the commit was successful, False otherwise. + """ return True - def delete_client(self, key: str) -> bool: - user = self.get_client_by_api_key(key) - if user: - item_id = self.get_item_id(user) - self.update_item(item_id, Client(-1, api_key="revoked")) + +class JsonDB(AbstractDB): + """Database implementation using JSON files.""" + + def __init__(self, name="clients", subfolder="hivemind-core"): + self._db: Dict[int, ClientDict] = JsonStorageXDG(name, subfolder=subfolder) + + def add_item(self, client: Client) -> bool: + """ + Add a client to the JSON database. + + Args: + client: The client to be added. + + Returns: + True if the addition was successful, False otherwise. + """ + self._db[client.client_id] = client.__dict__ + return True + + def delete_item(self, client: Client) -> bool: + """ + Delete a client from the JSON database. + + Args: + client: The client to be deleted. + + Returns: + True if the deletion was successful, False otherwise. + """ + if client.client_id in self._db: + self._db.pop(client.client_id) return True return False - def change_key(self, old_key: str, new_key: str) -> bool: - user = self.get_client_by_api_key(old_key) - if user is None: + def search_by_value(self, key: str, val: Union[str, bool, int, float]) -> List[Client]: + """ + Search for clients by a specific key-value pair in the JSON database. + + Args: + key: The key to search by. + val: The value to search for. + + Returns: + A list of clients that match the search criteria. + """ + res = [] + if key == "client_id": + v = self._db.get(val) + if v: + res.append(cast2client(v)) + else: + for client in self._db.values(): + v = client.get(key) + if v == val: + res.append(cast2client(client)) + return res + + def __len__(self) -> int: + """ + Get the number of clients in the database. + + Returns: + The number of clients in the database. + """ + return len(self._db) + + def __iter__(self) -> Iterable['Client']: + """ + Iterate over all clients in the JSON database. + + Returns: + An iterator over the clients in the database. + """ + for item in self._db.values(): + yield Client.deserialize(item) + + def commit(self) -> bool: + """ + Commit changes to the JSON database. + + Returns: + True if the commit was successful, False otherwise. + """ + try: + self._db.store() + return True + except Exception as e: + LOG.error(f"Failed to save {self._db.path}") return False - item_id = self.get_item_id(user) - user["api_key"] = new_key - self.update_item(item_id, user) - return True - def change_crypto_key(self, api_key: str, new_key: str) -> bool: - user = self.get_client_by_api_key(api_key) - if user is None: + +class RedisDB(AbstractDB): + """Database implementation using Redis with RediSearch support.""" + + def __init__(self, host: str = "127.0.0.1", port: int = 6379, redis_db: int = 0): + """ + Initialize the RedisDB connection. + + Args: + host: Redis server host. + port: Redis server port. + redis_db: Redis database index. + """ + if redis is None: + raise ImportError("pip install redis") + self.redis = redis.StrictRedis(host=host, port=port, db=redis_db, decode_responses=True) + + def add_item(self, client: Client) -> bool: + """ + Add a client to Redis and RediSearch. + + Args: + client: The client to be added. + + Returns: + True if the addition was successful, False otherwise. + """ + item_key = f"client:{client.client_id}" + serialized_data: str = client.serialize() + + try: + # Store data in Redis + self.redis.set(item_key, serialized_data) + return True + except Exception as e: + LOG.error(f"Failed to add client to Redis/RediSearch: {e}") return False - item_id = self.get_item_id(user) - user["crypto_key"] = new_key - self.update_item(item_id, user) - return True - def get_crypto_key(self, api_key: str) -> Optional[str]: - user = self.get_client_by_api_key(api_key) - if user is None: - return None - return user["crypto_key"] + def delete_item(self, client: Client) -> bool: + """ + Delete a client from Redis and RediSearch. - def get_password(self, api_key: str) -> Optional[str]: - user = self.get_client_by_api_key(api_key) - if user is None: - return None - return user["password"] + Args: + client: The client to be deleted. - def change_name(self, new_name: str, key: str) -> bool: - user = self.get_client_by_api_key(key) - if user is None: + Returns: + True if the deletion was successful, False otherwise. + """ + item_key = f"client:{client.client_id}" + try: + self.redis.delete(item_key) + return True + except Exception as e: + LOG.error(f"Failed to delete client from Redis: {e}") return False - item_id = self.get_item_id(user) - user["name"] = new_name - self.update_item(item_id, user) - return True - def change_blacklist(self, blacklist: Union[str, Dict[str, Any]], key: str) -> bool: - if isinstance(blacklist, dict): - blacklist = json.dumps(blacklist) + def search_by_value(self, key: str, val: Union[str, bool, int, float]) -> List[Client]: + """ + Search for clients by a specific key-value pair in Redis. + + Args: + key: The key to search by. + val: The value to search for. + + Returns: + A list of clients that match the search criteria. + """ + res = [] + for client_id in self.redis.scan_iter(f"client:*"): + client_data = self.redis.get(client_id) + client = cast2client(client_data) + if hasattr(client, key) and getattr(client, key) == val: + res.append(client) + return res + + def __len__(self) -> int: + """ + Get the number of items in the Redis database. + + Returns: + The number of clients in the database. + """ + return len(self.redis.keys("client:*")) + + def __iter__(self) -> Iterable['Client']: + """ + Iterate over all clients in Redis. + + Returns: + An iterator over the clients in the database. + """ + for client_id in self.redis.scan_iter(f"client:*"): + yield cast2client(self.redis.get(client_id)) + + +class ClientDatabase: + valid_backends = ["json", "redis"] + + def __init__(self, backend="json", **backend_kwargs): + """ + Initialize the client database with the specified backend. + """ + backend_kwargs = backend_kwargs or {} + if backend not in self.valid_backends: + raise NotImplementedError(f"{backend} not supported, choose one of {self.valid_backends}") + + if backend == "json": + self.db = JsonDB() + elif backend == "redis": + self.db = RedisDB(**backend_kwargs) + else: + raise NotImplementedError(f"{backend} not supported, valid databases: {self.valid_backends}") + + def delete_client(self, key: str) -> bool: user = self.get_client_by_api_key(key) - if user is None: - return False - item_id = self.get_item_id(user) - user["blacklist"] = blacklist - self.update_item(item_id, user) - return True + if user: + return self.db.delete_item(user) + return False - def get_blacklist_by_api_key(self, api_key: str): - search = self.search_by_value("api_key", api_key) - if len(search): - return search[0]["blacklist"] - return None + def get_clients_by_name(self, name: str) -> List[Client]: + return self.db.search_by_value("name", name) - @cast_to_client_obj() def get_client_by_api_key(self, api_key: str) -> Optional[Client]: - search = self.search_by_value("api_key", api_key) + search: List[Client] = self.db.search_by_value("api_key", api_key) if len(search): return search[0] return None - @cast_to_client_obj() - def get_clients_by_name(self, name: str) -> List[Client]: - return self.search_by_value("name", name) - - @cast_to_client_obj() - def add_client( - self, - name: str, - key: str = "", - admin: bool = False, - blacklist: Optional[Dict[str, Any]] = None, - allowed_types: Optional[List[str]] = None, - crypto_key: Optional[str] = None, - password: Optional[str] = None, - ) -> Client: - user = self.get_client_by_api_key(key) - item_id = self.get_item_id(user) + def add_client(self, + name: str, + key: str = "", + admin: bool = False, + intent_blacklist: Optional[List[str]] = None, + skill_blacklist: Optional[List[str]] = None, + message_blacklist: Optional[List[str]] = None, + allowed_types: Optional[List[str]] = None, + crypto_key: Optional[str] = None, + password: Optional[str] = None) -> bool: if crypto_key is not None: crypto_key = crypto_key[:16] - if item_id >= 0: + + user = self.get_client_by_api_key(key) + if user: + # Update the existing client object directly if name: - user["name"] = name - if blacklist: - user["blacklist"] = blacklist + user.name = name + if intent_blacklist: + user.intent_blacklist = intent_blacklist + if skill_blacklist: + user.skill_blacklist = skill_blacklist + if message_blacklist: + user.message_blacklist = message_blacklist if allowed_types: - user["allowed_types"] = allowed_types + user.allowed_types = allowed_types if admin is not None: - user["is_admin"] = admin + user.is_admin = admin if crypto_key: - user["crypto_key"] = crypto_key + user.crypto_key = crypto_key if password: - user["password"] = password - self.update_item(item_id, user) - else: - user = Client( - api_key=key, - name=name, - blacklist=blacklist, - crypto_key=crypto_key, - client_id=self.total_clients() + 1, - is_admin=admin, - password=password, - allowed_types=allowed_types, - ) - self.add_item(user) - return user + user.password = password + return self.db.update_item(user) + + user = Client( + api_key=key, + name=name, + intent_blacklist=intent_blacklist, + skill_blacklist=skill_blacklist, + message_blacklist=message_blacklist, + crypto_key=crypto_key, + client_id=self.total_clients() + 1, + is_admin=admin, + password=password, + allowed_types=allowed_types, + ) + return self.db.add_item(user) def total_clients(self) -> int: - return len(self) + return len(self.db) def __enter__(self): """Context handler""" return self + def __iter__(self) -> Iterable[Client]: + yield from self.db + def __exit__(self, _type, value, traceback): """Commits changes and Closes the session""" try: - self.commit() + self.db.commit() except Exception as e: LOG.error(e) diff --git a/hivemind_core/protocol.py b/hivemind_core/protocol.py index 67c39ad..f1bc09a 100644 --- a/hivemind_core/protocol.py +++ b/hivemind_core/protocol.py @@ -253,6 +253,7 @@ class HiveMindListenerProtocol: require_crypto: bool = True # throw error if crypto key not available handshake_enabled: bool = True # generate a key per session if not pre-shared identity: Optional[NodeIdentity] = None + db: Optional[ClientDatabase] = None # below are optional callbacks to handle payloads # receives the payload + HiveMindClient that sent it escalate_callback = None # slave asked to escalate payload @@ -262,8 +263,9 @@ class HiveMindListenerProtocol: mycroft_bus_callback = None # slave asked to inject payload into mycroft bus shared_bus_callback = None # passive sharing of slave device bus (info) - def bind(self, websocket, bus, identity): + def bind(self, websocket, bus, identity, db: ClientDatabase): self.identity = identity + self.db = db websocket.protocol = self self.internal_protocol = HiveMindListenerInternalProtocol(bus) self.internal_protocol.register_bus_handlers() @@ -755,10 +757,9 @@ def _update_blacklist(self, message: Message, client: HiveMindClientConnection): message.context["session"] = client.sess.serialize() # update blacklist from db, to account for changes without requiring a restart - with ClientDatabase() as users: - user = users.get_client_by_api_key(client.key) - client.skill_blacklist = user.blacklist.get("skills", []) - client.intent_blacklist = user.blacklist.get("intents", []) + user = self.db.get_client_by_api_key(client.key) + client.skill_blacklist = user.skill_blacklist + client.intent_blacklist = user.intent_blacklist # inject client specific blacklist into session if "blacklisted_skills" not in message.context["session"]: diff --git a/hivemind_core/scripts.py b/hivemind_core/scripts.py index cf40345..9d307b9 100644 --- a/hivemind_core/scripts.py +++ b/hivemind_core/scripts.py @@ -19,7 +19,24 @@ def hmcore_cmds(): @click.option("--access-key", required=False, type=str) @click.option("--password", required=False, type=str) @click.option("--crypto-key", required=False, type=str) -def add_client(name, access_key, password, crypto_key): +@click.option( + "--db-backend", + type=click.Choice(['redis', 'json'], case_sensitive=False), + default='json', + help="Select the database backend to use. Options: redis, json." +) +@click.option( + "--redis-host", + default="localhost", + help="Host for Redis (if selected). Default is localhost." +) +@click.option( + "--redis-port", + default=6379, + help="Port for Redis (if selected). Default is 6379." +) +def add_client(name, access_key, password, crypto_key, + db_backend, redis_host, redis_port): key = crypto_key if key: print( @@ -36,18 +53,26 @@ def add_client(name, access_key, password, crypto_key): key = os.urandom(8).hex() password = password or os.urandom(16).hex() - access_key = access_key or os.urandom(16).hex() - with ClientDatabase() as db: + + kwargs = {"backend": db_backend} + if db_backend == "redis": + kwargs["host"] = redis_host + kwargs["port"] = redis_port + with ClientDatabase(**kwargs) as db: name = name or f"HiveMind-Node-{db.total_clients()}" - db.add_client(name, access_key, crypto_key=key, password=password) + print(f"Database backend: {db.db.__class__.__name__}") + success = db.add_client(name, access_key, crypto_key=key, password=password) + if not success: + raise ValueError(f"Error adding User to database: {name}") # verify user = db.get_client_by_api_key(access_key) - node_id = db.get_item_id(user) + if user is None: + raise ValueError(f"User not found: {name}") print("Credentials added to database!\n") - print("Node ID:", node_id) + print("Node ID:", user.client_id) print("Friendly Name:", name) print("Access Key:", access_key) print("Password:", password) @@ -61,7 +86,29 @@ def add_client(name, access_key, password, crypto_key): @hmcore_cmds.command(help="allow message types sent from a client", name="allow-msg") @click.argument("msg_type", required=True, type=str) @click.argument("node_id", required=False, type=int) -def allow_msg(msg_type, node_id): +@click.option( + "--db-backend", + type=click.Choice(['redis', 'json'], case_sensitive=False), + default='json', + help="Select the database backend to use. Options: redis, json." +) +@click.option( + "--redis-host", + default="localhost", + help="Host for Redis (if selected). Default is localhost." +) +@click.option( + "--redis-port", + default=6379, + help="Port for Redis (if selected). Default is 6379." +) +def allow_msg(msg_type, node_id, + db_backend, redis_host, redis_port): + kwargs = {"backend": db_backend} + if db_backend == "redis": + kwargs["host"] = redis_host + kwargs["port"] = redis_port + if not node_id: # list clients and prompt for id using rich table = Table(title="HiveMind Clients") @@ -69,14 +116,14 @@ def allow_msg(msg_type, node_id): table.add_column("Name", style="magenta") table.add_column("Allowed Msg Types", style="yellow") _choices = [] - for client in ClientDatabase(): - if client["client_id"] != -1: + for client in ClientDatabase(**kwargs): + if client.client_id != -1: table.add_row( - str(client["client_id"]), - client["name"], - str(client.get("allowed_types", [])), + str(client.client_id), + client.name, + str(client.allowed_types), ) - _choices.append(str(client["client_id"])) + _choices.append(str(client.client_id)) if not _choices: print("No clients found!") @@ -95,19 +142,15 @@ def allow_msg(msg_type, node_id): else: node_id = _choices[0] - with ClientDatabase() as db: + with ClientDatabase(**kwargs) as db: for client in db: - if client["client_id"] == int(node_id): - allowed_types = client.get("allowed_types", []) - if msg_type in allowed_types: - print(f"Client {client['name']} already allowed '{msg_type}'") + if client.client_id == int(node_id): + if msg_type in client.allowed_types: + print(f"Client {client.name} already allowed '{msg_type}'") exit() - - allowed_types.append(msg_type) - client["allowed_types"] = allowed_types - item_id = db.get_item_id(client) - db.update_item(item_id, client) - print(f"Allowed '{msg_type}' for {client['name']}") + client.allowed_types.append(msg_type) + db.update_item(client) + print(f"Allowed '{msg_type}' for {client.name}") break @@ -115,8 +158,29 @@ def allow_msg(msg_type, node_id): help="remove credentials for a client (numeric unique ID)", name="delete-client" ) @click.argument("node_id", required=True, type=int) -def delete_client(node_id): - with ClientDatabase() as db: +@click.option( + "--db-backend", + type=click.Choice(['redis', 'json'], case_sensitive=False), + default='json', + help="Select the database backend to use. Options: redis, json." +) +@click.option( + "--redis-host", + default="localhost", + help="Host for Redis (if selected). Default is localhost." +) +@click.option( + "--redis-port", + default=6379, + help="Port for Redis (if selected). Default is 6379." +) +def delete_client(node_id, + db_backend, redis_host, redis_port): + kwargs = {"backend": db_backend} + if db_backend == "redis": + kwargs["host"] = redis_host + kwargs["port"] = redis_port + with ClientDatabase(**kwargs) as db: for x in db: if x["client_id"] == int(node_id): item_id = db.get_item_id(x) @@ -133,7 +197,23 @@ def delete_client(node_id): @hmcore_cmds.command(help="list clients and credentials", name="list-clients") -def list_clients(): +@click.option( + "--db-backend", + type=click.Choice(['redis', 'json'], case_sensitive=False), + default='json', + help="Select the database backend to use. Options: redis, json." +) +@click.option( + "--redis-host", + default="localhost", + help="Host for Redis (if selected). Default is localhost." +) +@click.option( + "--redis-port", + default=6379, + help="Port for Redis (if selected). Default is 6379." +) +def list_clients(db_backend, redis_host, redis_port): console = Console() table = Table(title="HiveMind Credentials:") table.add_column("ID", justify="center") @@ -142,7 +222,11 @@ def list_clients(): table.add_column("Password", justify="center") table.add_column("Crypto Key", justify="center") - with ClientDatabase() as db: + kwargs = {"backend": db_backend} + if db_backend == "redis": + kwargs["host"] = redis_host + kwargs["port"] = redis_port + with ClientDatabase(**kwargs) as db: for x in db: if x["client_id"] != -1: table.add_row( @@ -186,6 +270,22 @@ def list_clients(): type=str, default="hivemind", ) +@click.option( + "--db-backend", + type=click.Choice(['redis', 'json'], case_sensitive=False), + default='json', + help="Select the database backend to use. Options: redis, json." +) +@click.option( + "--redis-host", + default="localhost", + help="Host for Redis (if selected). Default is localhost." +) +@click.option( + "--redis-port", + default=6379, + help="Port for Redis (if selected). Default is 6379." +) def listen( ovos_bus_address: str, ovos_bus_port: int, @@ -194,6 +294,7 @@ def listen( ssl: bool, cert_dir: str, cert_name: str, + db_backend, redis_host, redis_port ): from hivemind_core.service import HiveMindService @@ -210,8 +311,15 @@ def listen( "cert_name": cert_name, } + kwargs = {"backend": db_backend} + if db_backend == "redis": + kwargs["host"] = redis_host + kwargs["port"] = redis_port + service = HiveMindService( - ovos_bus_config=ovos_bus_config, websocket_config=websocket_config + ovos_bus_config=ovos_bus_config, + websocket_config=websocket_config, + db=ClientDatabase(**kwargs) ) service.run() @@ -219,7 +327,29 @@ def listen( @hmcore_cmds.command(help="blacklist skills from being triggered by a client", name="blacklist-skill") @click.argument("skill_id", required=True, type=str) @click.argument("node_id", required=False, type=int) -def blacklist_skill(skill_id, node_id): +@click.option( + "--db-backend", + type=click.Choice(['redis', 'json'], case_sensitive=False), + default='json', + help="Select the database backend to use. Options: redis, json." +) +@click.option( + "--redis-host", + default="localhost", + help="Host for Redis (if selected). Default is localhost." +) +@click.option( + "--redis-port", + default=6379, + help="Port for Redis (if selected). Default is 6379." +) +def blacklist_skill(skill_id, node_id, + db_backend, redis_host, redis_port): + kwargs = {"backend": db_backend} + if db_backend == "redis": + kwargs["host"] = redis_host + kwargs["port"] = redis_port + if not node_id: # list clients and prompt for id using rich table = Table(title="HiveMind Clients") @@ -227,14 +357,14 @@ def blacklist_skill(skill_id, node_id): table.add_column("Name", style="magenta") table.add_column("Allowed Msg Types", style="yellow") _choices = [] - for client in ClientDatabase(): - if client["client_id"] != -1: + for client in ClientDatabase(**kwargs): + if client.client_id != -1: table.add_row( - str(client["client_id"]), - client["name"], - str(client.get("allowed_types", [])), + str(client.client_id), + client.name, + str(client.allowed_types), ) - _choices.append(str(client["client_id"])) + _choices.append(str(client.client_id)) if not _choices: print("No clients found!") @@ -253,26 +383,45 @@ def blacklist_skill(skill_id, node_id): else: node_id = _choices[0] - with ClientDatabase() as db: + with ClientDatabase(**kwargs) as db: for client in db: - if client["client_id"] == int(node_id): - blacklist = client.get("blacklist", {"messages": [], "skills": [], "intents": []}) - if skill_id in blacklist["skills"]: - print(f"Client {client['name']} already blacklisted '{skill_id}'") + if client.client_id == int(node_id): + if skill_id in client.skill_blacklist: + print(f"Client {client.name} already blacklisted '{skill_id}'") exit() - blacklist["skills"].append(skill_id) - client["blacklist"] = blacklist - item_id = db.get_item_id(client) - db.update_item(item_id, client) - print(f"Blacklisted '{skill_id}' for {client['name']}") + client.skill_blacklist.append(skill_id) + db.update_item(client) + print(f"Blacklisted '{skill_id}' for {client.name}") break @hmcore_cmds.command(help="remove skills from a client blacklist", name="unblacklist-skill") @click.argument("skill_id", required=True, type=str) @click.argument("node_id", required=False, type=int) -def unblacklist_skill(skill_id, node_id): +@click.option( + "--db-backend", + type=click.Choice(['redis', 'json'], case_sensitive=False), + default='json', + help="Select the database backend to use. Options: redis, json." +) +@click.option( + "--redis-host", + default="localhost", + help="Host for Redis (if selected). Default is localhost." +) +@click.option( + "--redis-port", + default=6379, + help="Port for Redis (if selected). Default is 6379." +) +def unblacklist_skill(skill_id, node_id, + db_backend, redis_host, redis_port): + kwargs = {"backend": db_backend} + if db_backend == "redis": + kwargs["host"] = redis_host + kwargs["port"] = redis_port + if not node_id: # list clients and prompt for id using rich table = Table(title="HiveMind Clients") @@ -280,14 +429,14 @@ def unblacklist_skill(skill_id, node_id): table.add_column("Name", style="magenta") table.add_column("Allowed Msg Types", style="yellow") _choices = [] - for client in ClientDatabase(): - if client["client_id"] != -1: + for client in ClientDatabase(**kwargs): + if client.client_id != -1: table.add_row( - str(client["client_id"]), - client["name"], - str(client.get("allowed_types", [])), + str(client.client_id), + client.name, + str(client.allowed_types), ) - _choices.append(str(client["client_id"])) + _choices.append(str(client.client_id)) if not _choices: print("No clients found!") @@ -306,26 +455,44 @@ def unblacklist_skill(skill_id, node_id): else: node_id = _choices[0] - with ClientDatabase() as db: + with ClientDatabase(**kwargs) as db: for client in db: - if client["client_id"] == int(node_id): - blacklist = client.get("blacklist", {"messages": [], "skills": [], "intents": []}) - if skill_id not in blacklist["skills"]: - print(f"'{skill_id}' is not blacklisted for client {client['name']}") + if client.client_id == int(node_id): + if skill_id not in client.skill_blacklist: + print(f"'{skill_id}' is not blacklisted for client {client.name}") exit() - - blacklist["skills"].pop(skill_id) - client["blacklist"] = blacklist - item_id = db.get_item_id(client) - db.update_item(item_id, client) - print(f"Blacklisted '{skill_id}' for {client['name']}") + client.skill_blacklist.remove(skill_id) + db.update_item(client) + print(f"Blacklisted '{skill_id}' for {client.name}") break @hmcore_cmds.command(help="blacklist intents from being triggered by a client", name="blacklist-intent") @click.argument("intent_id", required=True, type=str) @click.argument("node_id", required=False, type=int) -def blacklist_intent(intent_id, node_id): +@click.option( + "--db-backend", + type=click.Choice(['redis', 'json'], case_sensitive=False), + default='json', + help="Select the database backend to use. Options: redis, json." +) +@click.option( + "--redis-host", + default="localhost", + help="Host for Redis (if selected). Default is localhost." +) +@click.option( + "--redis-port", + default=6379, + help="Port for Redis (if selected). Default is 6379." +) +def blacklist_intent(intent_id, node_id, + db_backend, redis_host, redis_port): + kwargs = {"backend": db_backend} + if db_backend == "redis": + kwargs["host"] = redis_host + kwargs["port"] = redis_port + if not node_id: # list clients and prompt for id using rich table = Table(title="HiveMind Clients") @@ -333,14 +500,14 @@ def blacklist_intent(intent_id, node_id): table.add_column("Name", style="magenta") table.add_column("Allowed Msg Types", style="yellow") _choices = [] - for client in ClientDatabase(): - if client["client_id"] != -1: + for client in ClientDatabase(**kwargs): + if client.client_id != -1: table.add_row( - str(client["client_id"]), - client["name"], - str(client.get("allowed_types", [])), + str(client.client_id), + client.name, + str(client.allowed_types), ) - _choices.append(str(client["client_id"])) + _choices.append(str(client.client_id)) if not _choices: print("No clients found!") @@ -359,26 +526,44 @@ def blacklist_intent(intent_id, node_id): else: node_id = _choices[0] - with ClientDatabase() as db: + with ClientDatabase(**kwargs) as db: for client in db: - if client["client_id"] == int(node_id): - blacklist = client.get("blacklist", {"messages": [], "skills": [], "intents": []}) - if intent_id in blacklist["intents"]: - print(f"Client {client['name']} already blacklisted '{intent_id}'") + if client.client_id == int(node_id): + if intent_id in client.intent_blacklist: + print(f"Client {client.name} already blacklisted '{intent_id}'") exit() - - blacklist["intents"].append(intent_id) - client["blacklist"] = blacklist - item_id = db.get_item_id(client) - db.update_item(item_id, client) - print(f"Blacklisted '{intent_id}' for {client['name']}") + client.intent_blacklist.append(intent_id) + db.update_item(client) + print(f"Blacklisted '{intent_id}' for {client.name}") break @hmcore_cmds.command(help="remove intents from a client blacklist", name="unblacklist-intent") @click.argument("intent_id", required=True, type=str) @click.argument("node_id", required=False, type=int) -def unblacklist_intent(intent_id, node_id): +@click.option( + "--db-backend", + type=click.Choice(['redis', 'json'], case_sensitive=False), + default='json', + help="Select the database backend to use. Options: redis, json." +) +@click.option( + "--redis-host", + default="localhost", + help="Host for Redis (if selected). Default is localhost." +) +@click.option( + "--redis-port", + default=6379, + help="Port for Redis (if selected). Default is 6379." +) +def unblacklist_intent(intent_id, node_id, + db_backend, redis_host, redis_port): + kwargs = {"backend": db_backend} + if db_backend == "redis": + kwargs["host"] = redis_host + kwargs["port"] = redis_port + if not node_id: # list clients and prompt for id using rich table = Table(title="HiveMind Clients") @@ -386,14 +571,14 @@ def unblacklist_intent(intent_id, node_id): table.add_column("Name", style="magenta") table.add_column("Allowed Msg Types", style="yellow") _choices = [] - for client in ClientDatabase(): - if client["client_id"] != -1: + for client in ClientDatabase(**kwargs): + if client.client_id != -1: table.add_row( - str(client["client_id"]), - client["name"], - str(client.get("allowed_types", [])), + str(client.client_id), + client.name, + str(client.allowed_types), ) - _choices.append(str(client["client_id"])) + _choices.append(str(client.client_id)) if not _choices: print("No clients found!") @@ -412,19 +597,15 @@ def unblacklist_intent(intent_id, node_id): else: node_id = _choices[0] - with ClientDatabase() as db: + with ClientDatabase(**kwargs) as db: for client in db: - if client["client_id"] == int(node_id): - blacklist = client.get("blacklist", {"messages": [], "skills": [], "intents": []}) - if intent_id not in blacklist["intents"]: - print(f" '{intent_id}' not blacklisted for Client {client['name']} ") + if client.client_id == int(node_id): + if intent_id not in client.intent_blacklist: + print(f" '{intent_id}' not blacklisted for Client {client.name} ") exit() - - blacklist["intents"].pop(intent_id) - client["blacklist"] = blacklist - item_id = db.get_item_id(client) - db.update_item(item_id, client) - print(f"Blacklisted '{intent_id}' for {client['name']}") + client.intent_blacklist.remove(intent_id) + db.update_item(client) + print(f"Unblacklisted '{intent_id}' for {client.name}") break diff --git a/hivemind_core/service.py b/hivemind_core/service.py index b070952..09cdc74 100644 --- a/hivemind_core/service.py +++ b/hivemind_core/service.py @@ -99,6 +99,7 @@ def on_stopping(): class MessageBusEventHandler(WebSocketHandler): protocol: Optional[HiveMindListenerProtocol] = None + db: Optional[ClientDatabase] = None @staticmethod def decode_auth(auth) -> Tuple[str, str]: @@ -134,42 +135,46 @@ def open(self): handshake=handshake, loop=self.protocol.loop, ) + if self.db is None: + LOG.error("HiveMind database not initialized, can't validate connection") + self.protocol.handle_invalid_key_connected(self.client) + self.close() + return + + user = self.db.get_client_by_api_key(key) + if not user: + LOG.error("Client provided an invalid api key") + self.protocol.handle_invalid_key_connected(self.client) + self.close() + return + + self.client.crypto_key = user.crypto_key + self.client.msg_blacklist = user.message_blacklist + self.client.skill_blacklist = user.skill_blacklist + self.client.intent_blacklist = user.intent_blacklist + self.client.allowed_types = user.allowed_types + self.client.can_broadcast = user.can_broadcast + self.client.can_propagate = user.can_propagate + self.client.can_escalate = user.can_escalate + if user.password: + # pre-shared password to derive aes_key + self.client.pswd_handshake = PasswordHandShake(user.password) + + self.client.node_type = HiveMindNodeType.NODE # TODO . placeholder - with ClientDatabase() as users: - user = users.get_client_by_api_key(key) - if not user: - LOG.error("Client provided an invalid api key") - self.protocol.handle_invalid_key_connected(self.client) - self.close() - return - - self.client.crypto_key = user.crypto_key - self.client.msg_blacklist = user.blacklist.get("messages", []) - self.client.skill_blacklist = user.blacklist.get("skills", []) - self.client.intent_blacklist = user.blacklist.get("intents", []) - self.client.allowed_types = user.allowed_types - self.client.can_broadcast = user.can_broadcast - self.client.can_propagate = user.can_propagate - self.client.can_escalate = user.can_escalate - if user.password: - # pre-shared password to derive aes_key - self.client.pswd_handshake = PasswordHandShake(user.password) - - self.client.node_type = HiveMindNodeType.NODE # TODO . placeholder - - if ( - not self.client.crypto_key - and not self.protocol.handshake_enabled - and self.protocol.require_crypto - ): - LOG.error( - "No pre-shared crypto key for client and handshake disabled, " - "but configured to require crypto!" - ) - # clients requiring handshake support might fail here - self.protocol.handle_invalid_protocol_version(self.client) - self.close() - return + if ( + not self.client.crypto_key + and not self.protocol.handshake_enabled + and self.protocol.require_crypto + ): + LOG.error( + "No pre-shared crypto key for client and handshake disabled, " + "but configured to require crypto!" + ) + # clients requiring handshake support might fail here + self.protocol.handle_invalid_protocol_version(self.client) + self.close() + return self.protocol.handle_new_client(self.client) # self.write_message(Message("connected").serialize()) @@ -197,6 +202,7 @@ def __init__( protocol=HiveMindListenerProtocol, bus=None, ws_handler=MessageBusEventHandler, + db: ClientDatabase = None ): websocket_config = websocket_config or Configuration().get( "hivemind_websocket", {} @@ -208,8 +214,10 @@ def __init__( on_error=error_hook, on_stopping=stopping_hook, ) + self.db = db self._proto = protocol self._ws_handler = ws_handler + self._ws_handler.db = db if bus: self.bus = bus else: @@ -256,7 +264,7 @@ def run(self): loop = ioloop.IOLoop.current() self.protocol = self._proto(loop=loop) - self.protocol.bind(self._ws_handler, self.bus, self.identity) + self.protocol.bind(self._ws_handler, self.bus, self.identity, self.db) self.status.bind(self.bus) self.status.set_started() diff --git a/setup.py b/setup.py index 155a11e..607e5a6 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def required(requirements_file): setup( - name="jarbas_hive_mind", + name="hivemind-core", version=get_version(), packages=["hivemind_core"], include_package_data=True, diff --git a/test/unittests/test_db.py b/test/unittests/test_db.py index fd8c1d5..88412f1 100644 --- a/test/unittests/test_db.py +++ b/test/unittests/test_db.py @@ -1,37 +1,182 @@ -import os -from unittest import TestCase - -from hivemind_core.database import ClientDatabase, Client - - -class TestDB(TestCase): - def test_add_entry(self): - key = os.urandom(8).hex() - access_key = os.urandom(16).hex() - password = None - - with ClientDatabase() as db: - n = db.total_clients() - name = f"HiveMind-Node-{n}" - user = db.add_client(name, access_key, crypto_key=key, password=password) - # verify data - self.assertTrue(isinstance(user, Client)) - self.assertEqual(user.name, name) - self.assertEqual(user.api_key, access_key) - - # test search entry in db - node_id = db.get_item_id(user) - self.assertEqual(node_id, n) - - user2 = db.get_client_by_api_key(access_key) - self.assertEqual(user, user2) - - for u in db.get_clients_by_name(name): - self.assertEqual(user.name, u.name) - - # test delete entry - db.delete_client(access_key) - node_id = db.get_item_id(user) - self.assertEqual(node_id, -1) - user = db.get_client_by_api_key(access_key) - self.assertIsNone(user) +import json +import unittest +from unittest.mock import patch, MagicMock + +from hivemind_core.database import Client, JsonDB, RedisDB, cast2client, ClientDatabase + + +class TestClient(unittest.TestCase): + + def test_client_creation(self): + client_data = { + "client_id": 1, + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": False + } + client = Client(**client_data) + self.assertEqual(client.client_id, 1) + self.assertEqual(client.api_key, "test_api_key") + self.assertEqual(client.name, "Test Client") + self.assertEqual(client.description, "A test client") + self.assertFalse(client.is_admin) + + def test_client_serialization(self): + client_data = { + "client_id": 1, + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": False + } + client = Client(**client_data) + serialized_data = client.serialize() + self.assertIsInstance(serialized_data, str) + self.assertIn('"client_id": 1', serialized_data) + + def test_client_deserialization(self): + client_data = { + "client_id": 1, + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": False + } + serialized_data = json.dumps(client_data) + client = Client.deserialize(serialized_data) + self.assertEqual(client.client_id, 1) + self.assertEqual(client.api_key, "test_api_key") + + def test_cast2client(self): + client_data = { + "client_id": 1, + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": False + } + client = Client(**client_data) + serialized_client = client.serialize() + deserialized_client = cast2client(serialized_client) + self.assertEqual(client, deserialized_client) + + client_list = [client, client] + deserialized_client_list = cast2client([serialized_client, serialized_client]) + self.assertEqual(client_list, deserialized_client_list) + + +class TestJsonDB(unittest.TestCase): + + def setUp(self): + self.db = JsonDB(name=".hivemind-test") + + def test_add_item(self): + client_data = { + "client_id": 1, + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": False + } + client = Client(**client_data) + self.db.add_item(client) + self.assertTrue(client.client_id in self.db._db) + + def test_delete_item(self): + client_data = { + "client_id": 1, + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": False + } + client = Client(**client_data) + self.db.add_item(client) + result = self.db.delete_item(client) + self.assertTrue(result) + + def test_search_by_value(self): + client_data = { + "client_id": 1, + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": False + } + client = Client(**client_data) + self.db.add_item(client) + clients = self.db.search_by_value("name", "Test Client") + self.assertEqual(len(clients), 1) + self.assertEqual(clients[0].name, "Test Client") + + +class TestRedisDB(unittest.TestCase): + + @patch('hivemind_core.database.redis.StrictRedis') + def setUp(self, MockRedis): + self.mock_redis = MagicMock() + MockRedis.return_value = self.mock_redis + self.db = RedisDB() + + def test_add_item(self): + client_data = { + "client_id": 1, + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": False + } + client = Client(**client_data) + self.db.add_item(client) + self.mock_redis.set.assert_called_once() + + def test_delete_item(self): + client_data = { + "client_id": 1, + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": False + } + client = Client(**client_data) + self.db.add_item(client) + result = self.db.delete_item(client) + self.assertTrue(result) + + +class TestClientDatabase(unittest.TestCase): + + def test_delete_client(self): + db = MagicMock() + db.delete_item.return_value = True + client_db = ClientDatabase(backend="json") + client_db.db = db + client_db.get_client_by_api_key = MagicMock() + client_db.get_client_by_api_key.return_value = Client(1, "A") + + result = client_db.delete_client("test_api_key") + self.assertTrue(result) + db.delete_item.assert_called_once() + + def test_get_clients_by_name(self): + db = MagicMock() + client_data = { + "client_id": 1, + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": False + } + client = Client(**client_data) + db.search_by_value.return_value = [client] + + client_db = ClientDatabase(backend="json") + client_db.db = db + clients = client_db.get_clients_by_name("Test Client") + self.assertEqual(len(clients), 1) + self.assertEqual(clients[0].name, "Test Client") + + +if __name__ == '__main__': + unittest.main() From 9693a2e9d70b9430574281a8afc9579baabad33d Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 20 Dec 2024 03:51:11 +0000 Subject: [PATCH 02/12] Update hivemind_core/service.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- hivemind_core/service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hivemind_core/service.py b/hivemind_core/service.py index 09cdc74..962c737 100644 --- a/hivemind_core/service.py +++ b/hivemind_core/service.py @@ -136,7 +136,8 @@ def open(self): loop=self.protocol.loop, ) if self.db is None: - LOG.error("HiveMind database not initialized, can't validate connection") + LOG.error("Database connection not initialized. Please ensure database configuration is correct.") + LOG.debug(f"Client {client.peer} connection attempt failed due to missing database connection") self.protocol.handle_invalid_key_connected(self.client) self.close() return From 982b41d1bd071f710af49ad532d9f5f3b8cc9ac2 Mon Sep 17 00:00:00 2001 From: miro Date: Fri, 20 Dec 2024 13:53:23 +0000 Subject: [PATCH 03/12] add sqlite support Update README.md --- README.md | 307 ++++++++++++++++++++++++++++++-------- hivemind_core/database.py | 174 ++++++++++++++++++++- hivemind_core/scripts.py | 201 +++++++++++++++++++------ 3 files changed, 568 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index cf493da..35af18e 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,75 @@ -

- -

+# HiveMind Core -HiveMind is a community-developed superset or extension of [OpenVoiceOS](https://github.com/OpenVoiceOS/), the open-source voice assistant. +HiveMind is an extension of [OpenVoiceOS (OVOS)](https://github.com/OpenVoiceOS/), the open-source voice assistant +platform. It enables you to extend a single instance of `ovos-core` across multiple devices, even those with limited +hardware capabilities that can't typically run OVOS. -With HiveMind, you can extend one (or more, but usually just one!) instance of ovos-core to as many devices as you want, including devices that can't ordinarily run OpenVoiceOS! +Demo videos in [youtube](https://www.youtube.com/channel/UCYoV5kxp2zrH6pnoqVZpKSA/) -HiveMind's developers have successfully connected to OpenVoiceOS from a PinePhone, a 2009 MacBook, and a Raspberry Pi 0, among other devices. OpenVoiceOS itself usually runs on our desktop computers or our home servers, but you can use any OpenVoiceOS-branded device as your central unit. +--- -Work in progress documentation can be found in the [docs](https://jarbashivemind.github.io/HiveMind-community-docs) +## 🌟 Key Features -You can also join the [Hivemind Matrix chat](https://matrix.to/#/#jarbashivemind:matrix.org) for general news, support and chit chat +- **Expand Your Ecosystem**: Seamlessly connect lightweight or legacy devices as satellites to a central OVOS hub. +- **Centralized Control**: Manage and monitor all connected devices from a single hivemind-core instance. +- **Fine-Grained Permissions**: Control skills, intents, and message access per client. +- **Flexible Database Support**: Choose from JSON, SQLite, or Redis to fit your setup. +--- +## 📖 Documentation & Community -# Usage +- 📚 **Documentation**: [HiveMind Docs (WIP)](https://jarbashivemind.github.io/HiveMind-community-docs) +- 💬 **Chat**: Join the [HiveMind Matrix Chat](https://matrix.to/#/#jarbashivemind:matrix.org) for news, support, and + discussion. +--- + +## 🚀 Getting Started + +To get started, HiveMind Core provides a command-line interface (CLI) for managing clients, permissions, and +connections. + +### Installation + +```bash +pip install hivemind-core ``` + +### Adding a satellite + +Add credentials for each satellite device + +```bash +$ hivemind-core add-client --db-backend sqlite +Database backend: SQLiteDB +Credentials added to database! + +Node ID: 3 +Friendly Name: HiveMind-Node-2 +Access Key: 42caf3d2405075fb9e7a4e1ff44e4c4f +Password: 5ae486f7f1c26bd4645bd052e4af3ea3 +Encryption Key: f46351c54f61a715 +WARNING: Encryption Key is deprecated, only use if your client does not support password +``` + +**NOTE**: you will need to provide this info on the client devices in order to connect + +### Running the Server + +Start the HiveMind Core server to accept connections: + +```bash +$ hivemind-core listen --port 5678 +``` + +--- + +## 🛠️ Commands Overview + +HiveMind Core CLI supports the following commands: + +```bash $ hivemind-core --help Usage: hivemind-core [OPTIONS] COMMAND [ARGS]... @@ -24,96 +77,224 @@ Options: --help Show this message and exit. Commands: - add-client add credentials for a client - delete-client remove credentials for a client - list-clients list clients and credentials - listen start listening for HiveMind connections + add-client Add credentials for a client. + allow-msg Allow specific message types from a client. + blacklist-intent Block certain intents for a client. + blacklist-skill Block certain skills for a client. + delete-client Remove client credentials. + list-clients Display a list of registered clients. + listen Start listening for HiveMind connections. + unblacklist-intent Remove intents from a client's blacklist. + unblacklist-skill Remove skills from a client's blacklist. +``` +For detailed help on each command, use `--help` (e.g., `hivemind-core add-client --help`). -$ hivemind-core add-client --help -Usage: hivemind-core add-client [OPTIONS] [NAME] [ACCESS_KEY] [PASSWORD] - [CRYPTO_KEY] +--- - add credentials for a client +### `add-client` -Options: - --help Show this message and exit. +Add credentials for a new client that will connect to the HiveMind instance. +```bash +$ hivemind-core add-client --name "satellite_1" --access-key "mykey123" --password "mypass" --db-backend json +``` +- **When to use**: + Use this command when setting up a new HiveMind client device (e.g., a Raspberry Pi or another satellite). You’ll need + to provide the credentials for secure communication. -$ hivemind-core listen --help -Usage: hivemind-core listen [OPTIONS] +--- - start listening for HiveMind connections +### `list-clients` -Options: - --host TEXT HiveMind host - --port INTEGER HiveMind port number - --ovos_bus_address TEXT Open Voice OS bus address - --ovos_bus_port INTEGER Open Voice OS bus port - --ssl BOOLEAN use wss:// - --cert_dir TEXT HiveMind SSL certificate directory - --cert_name TEXT HiveMind SSL certificate file name - --help Show this message and exit. +List all the registered clients and their credentials. +```bash +$ hivemind-core list-clients --db-backend json +``` -$ hivemind-core delete-client --help -Usage: hivemind-core delete-client [OPTIONS] NODE_ID +- **When to use**: + Use this command to verify which clients are currently registered or to inspect their credentials. This is helpful for + debugging or managing connected devices. - remove credentials for a client +--- -Options: - --help Show this message and exit. +### `delete-client` +Remove a registered client from the HiveMind instance. -$ hivemind-core list-clients --help -Usage: hivemind-core list-clients [OPTIONS] +```bash +$ hivemind-core delete-client 1 +``` - list clients and credentials +- **When to use**: + Use this command to revoke access for a specific client. For instance, if a device is lost, no longer in use, or + compromised, you can remove it to maintain security. -Options: - --help Show this message and exit. +--- + +### `allow-msg` + +Allow specific message types to be sent by a client. +```bash +$ hivemind-core allow-msg "speak" ``` -By default HiveMind listens for the Open Voice OS bus on `127.0.0.1` which should not be changed when running as the same place. In some cases such as Kubernetes when the HiveMind Listener and Open Voice OS bus are in different pods, the HiveMind Listener should be able to connect to the pod address by using the `ovos_bus_address` and `ovos_bus_port` options. +- **When to use**: + This command is used to fine-tune the communication protocol by enabling specific message types. This is especially + useful in scenarios where certain clients should only perform limited actions (e.g., making another device speak via + TTS). -# Protocol +--- -| Protocol Version | 0 | 1 | -| -------------------- | --- | --- | -| json serialization | yes | yes | -| binary serialization | no | yes | -| pre-shared AES key | yes | yes | -| password handshake | no | yes | -| PGP handshake | no | yes | -| zlib compression | no | yes | +### `blacklist-skill` -some clients such as HiveMind-Js do not yet support protocol V1 +Prevent a specific skill from being triggered by a client. -# HiveMind components +```bash +$ hivemind-core blacklist-skill "skill-weather" 1 +``` + +- **When to use**: + Use this command to restrict a client from interacting with a particular skill. For example, a child’s device could be + restricted from accessing skills that are not age-appropriate. + +--- + +### `unblacklist-skill` + +Remove a skill from a client’s blacklist. -![](./resources/1m5s.svg) +```bash +$ hivemind-core unblacklist-skill "skill-weather" 1 +``` + +- **When to use**: + If restrictions are no longer needed, use this command to restore access to the blacklisted skill. + +--- -## Client Libraries +### `blacklist-intent` -- [HiveMind-websocket-client](https://github.com/JarbasHiveMind/hivemind_websocket_client) -- [HiveMindJs](https://github.com/JarbasHiveMind/HiveMind-js) +Block a specific intent from being triggered by a client. + +```bash +$ hivemind-core blacklist-intent "intent.check_weather" 1 +``` -## Terminals +- **When to use**: + Use this command when fine-grained control is needed to block individual intents for a specific client, especially in + environments with shared skills but different permission levels. -- [Remote Cli](https://github.com/OpenJarbas/HiveMind-cli) **\<-- USE THIS FIRST** +--- + +### `unblacklist-intent` + +Remove an intent from a client’s blacklist. + +```bash +$ hivemind-core unblacklist-intent "intent.check_weather" 1 +``` + +- **When to use**: + This command allows you to reinstate access to previously blocked intents. + +--- + +### `listen` + +Start the HiveMind instance to accept client connections. + +```bash +$ hivemind-core listen --ovos_bus_address "127.0.0.1" --port 5678 +``` + +- **When to use**: + Run this command on the central HiveMind instance (e.g., a server or desktop) to start listening for connections from + satellite devices. Configure host, port, and security options as needed. + +--- + +#### Running in Distributed Environments + +By default, HiveMind listens for the OpenVoiceOS bus on `127.0.0.1`. When running in distributed environments (e.g., +Kubernetes), use the `--ovos_bus_address` and `--ovos_bus_port` options to specify the bus address and port. + +--- + +## 📦 Database Backends + +HiveMind-Core supports multiple database backends to store client credentials and settings. Each has its own use case: + +| Backend | Use Case | Default Location | Command Line options | +|--------------------|------------------------------------------------|---------------------------------------|----------------------------------------------------| +| **JSON** (default) | Simple, file-based setup for local use | `~/.cache/hivemind-core/clients.json` | Configurable via `--db-name` and `--db-folder` | +| **SQLite** | Lightweight relational DB for single instances | `~/.cache/hivemind-core/clients.db` | Configurable via `--db-name` and `--db-folder` | +| **Redis** | Distributed, high-performance environments | `localhost:6379` | Configurable via `--redis-host` and `--redis-port` | + +**How to Choose?** + +- For **scalability** or multi-instance setups, use Redis. +- For **simplicity** or single-device environments, use JSON or SQLite. + +--- + +## 🔒 Protocol Support + +| Feature | Protocol v0 | Protocol v1 | +|----------------------|-------------|-------------| +| JSON serialization | ✅ | ✅ | +| Binary serialization | ❌ | ✅ | +| Pre-shared AES key | ✅ | ✅ | +| Password handshake | ❌ | ✅ | +| PGP handshake | ❌ | ✅ | +| Zlib compression | ❌ | ✅ | + +> **Note**: Some clients (e.g., HiveMind-JS) do not yet support Protocol v1. + +--- + +## 🧩 HiveMind Ecosystem + +### Core Components + +- **HiveMind Core** (this repository): The central hub for managing connections and routing messages between devices. + +### Client Libraries + +- [HiveMind WebSocket Client](https://github.com/JarbasHiveMind/hivemind_websocket_client) +- [HiveMind JS](https://github.com/JarbasHiveMind/HiveMind-js) + +### Terminals + +- [Remote CLI](https://github.com/OpenJarbas/HiveMind-cli) (**Recommended Starting Point**) - [Voice Satellite](https://github.com/OpenJarbas/HiveMind-voice-sat) - [Flask Chatroom](https://github.com/JarbasHiveMind/HiveMind-flask-template) -- [Webchat](https://github.com/OpenJarbas/HiveMind-webchat) +- [Web Chat](https://github.com/OpenJarbas/HiveMind-webchat) -## Bridges +### Bridges - [Mattermost Bridge](https://github.com/OpenJarbas/HiveMind_mattermost_bridge) - [HackChat Bridge](https://github.com/OpenJarbas/HiveMind-HackChatBridge) - [Twitch Bridge](https://github.com/OpenJarbas/HiveMind-twitch-bridge) - [DeltaChat Bridge](https://github.com/JarbasHiveMind/HiveMind-deltachat-bridge) -## Minds +### Minds + +- [NodeRed Integration](https://github.com/OpenJarbas/HiveMind-NodeRed) + +--- + +## 🤝 Contributing + +Contributions are welcome! + +--- + +## ⚖️ License + +HiveMind is open-source software, licensed under the [Apache 2.0 License](LICENSE). + -- [NodeRed](https://github.com/OpenJarbas/HiveMind-NodeRed) diff --git a/hivemind_core/database.py b/hivemind_core/database.py index 2b283dd..dcef3ef 100644 --- a/hivemind_core/database.py +++ b/hivemind_core/database.py @@ -1,16 +1,22 @@ import abc import json +import os.path from dataclasses import dataclass, field from typing import List, Dict, Union, Any, Optional, Iterable from json_database import JsonStorageXDG from ovos_utils.log import LOG +from ovos_utils.xdg_utils import xdg_data_home try: import redis except ImportError: redis = None +try: + import sqlite3 +except ImportError: + sqlite3 = None ClientDict = Dict[str, Union[str, int, float, List[str]]] ClientTypes = Union[None, 'Client', @@ -353,6 +359,167 @@ def commit(self) -> bool: return False +class SQLiteDB(AbstractDB): + """Database implementation using SQLite.""" + + def __init__(self, name="clients", subfolder="hivemind-core"): + """ + Initialize the SQLiteDB connection. + + Args: + db_path: Path to the SQLite database file. Default is in-memory. + """ + if sqlite3 is None: + raise ImportError("pip install sqlite3") + db_path = os.path.join(xdg_data_home(), subfolder, name + ".db") + os.makedirs(os.path.dirname(db_path), exist_ok=True) + + self.conn = sqlite3.connect(db_path) + self.conn.row_factory = sqlite3.Row + self._initialize_database() + + def _initialize_database(self): + """Initialize the database schema.""" + with self.conn: + self.conn.execute(""" + CREATE TABLE IF NOT EXISTS clients ( + client_id INTEGER PRIMARY KEY, + api_key TEXT NOT NULL, + name TEXT, + description TEXT, + is_admin BOOLEAN DEFAULT FALSE, + last_seen REAL DEFAULT -1, + intent_blacklist TEXT, + skill_blacklist TEXT, + message_blacklist TEXT, + allowed_types TEXT, + crypto_key TEXT, + password TEXT, + can_broadcast BOOLEAN DEFAULT TRUE, + can_escalate BOOLEAN DEFAULT TRUE, + can_propagate BOOLEAN DEFAULT TRUE + ) + """) + + def add_item(self, client: Client) -> bool: + """ + Add a client to the SQLite database. + + Args: + client: The client to be added. + + Returns: + True if the addition was successful, False otherwise. + """ + try: + with self.conn: + self.conn.execute(""" + INSERT OR REPLACE INTO clients ( + client_id, api_key, name, description, is_admin, + last_seen, intent_blacklist, skill_blacklist, + message_blacklist, allowed_types, crypto_key, password, + can_broadcast, can_escalate, can_propagate + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, ( + client.client_id, client.api_key, client.name, client.description, + client.is_admin, client.last_seen, + json.dumps(client.intent_blacklist), + json.dumps(client.skill_blacklist), + json.dumps(client.message_blacklist), + json.dumps(client.allowed_types), + client.crypto_key, client.password, + client.can_broadcast, client.can_escalate, client.can_propagate + )) + return True + except sqlite3.Error as e: + LOG.error(f"Failed to add client to SQLite: {e}") + return False + + def delete_item(self, client: Client) -> bool: + """ + Delete a client from the SQLite database. + + Args: + client: The client to be deleted. + + Returns: + True if the deletion was successful, False otherwise. + """ + try: + with self.conn: + self.conn.execute("DELETE FROM clients WHERE client_id = ?", (client.client_id,)) + return True + except sqlite3.Error as e: + LOG.error(f"Failed to delete client from SQLite: {e}") + return False + + def search_by_value(self, key: str, val: Union[str, bool, int, float]) -> List[Client]: + """ + Search for clients by a specific key-value pair in the SQLite database. + + Args: + key: The key to search by. + val: The value to search for. + + Returns: + A list of clients that match the search criteria. + """ + try: + with self.conn: + cur = self.conn.execute(f"SELECT * FROM clients WHERE {key} = ?", (val,)) + rows = cur.fetchall() + return [self._row_to_client(row) for row in rows] + except sqlite3.Error as e: + LOG.error(f"Failed to search clients in SQLite: {e}") + return [] + + def __len__(self) -> int: + """Get the number of clients in the database.""" + cur = self.conn.execute("SELECT COUNT(*) FROM clients") + return cur.fetchone()[0] + + def __iter__(self) -> Iterable['Client']: + """ + Iterate over all clients in the SQLite database. + + Returns: + An iterator over the clients in the database. + """ + cur = self.conn.execute("SELECT * FROM clients") + for row in cur: + yield self._row_to_client(row) + + def commit(self) -> bool: + """Commit changes to the SQLite database.""" + try: + self.conn.commit() + return True + except sqlite3.Error as e: + LOG.error(f"Failed to commit SQLite database: {e}") + return False + + @staticmethod + def _row_to_client(row: sqlite3.Row) -> Client: + """Convert a database row to a Client instance.""" + return Client( + client_id=row["client_id"], + api_key=row["api_key"], + name=row["name"], + description=row["description"], + is_admin=row["is_admin"], + last_seen=row["last_seen"], + intent_blacklist=json.loads(row["intent_blacklist"] or "[]"), + skill_blacklist=json.loads(row["skill_blacklist"] or "[]"), + message_blacklist=json.loads(row["message_blacklist"] or "[]"), + allowed_types=json.loads(row["allowed_types"] or "[]"), + crypto_key=row["crypto_key"], + password=row["password"], + can_broadcast=row["can_broadcast"], + can_escalate=row["can_escalate"], + can_propagate=row["can_propagate"] + ) + + class RedisDB(AbstractDB): """Database implementation using Redis with RediSearch support.""" @@ -368,6 +535,7 @@ def __init__(self, host: str = "127.0.0.1", port: int = 6379, redis_db: int = 0) if redis is None: raise ImportError("pip install redis") self.redis = redis.StrictRedis(host=host, port=port, db=redis_db, decode_responses=True) + # TODO - support for a proper search index def add_item(self, client: Client) -> bool: """ @@ -448,7 +616,7 @@ def __iter__(self) -> Iterable['Client']: class ClientDatabase: - valid_backends = ["json", "redis"] + valid_backends = ["json", "redis", "sqlite"] def __init__(self, backend="json", **backend_kwargs): """ @@ -459,9 +627,11 @@ def __init__(self, backend="json", **backend_kwargs): raise NotImplementedError(f"{backend} not supported, choose one of {self.valid_backends}") if backend == "json": - self.db = JsonDB() + self.db = JsonDB(**backend_kwargs) elif backend == "redis": self.db = RedisDB(**backend_kwargs) + elif backend == "sqlite": + self.db = SQLiteDB(**backend_kwargs) else: raise NotImplementedError(f"{backend} not supported, valid databases: {self.valid_backends}") diff --git a/hivemind_core/scripts.py b/hivemind_core/scripts.py index 9d307b9..95f63ed 100644 --- a/hivemind_core/scripts.py +++ b/hivemind_core/scripts.py @@ -21,22 +21,39 @@ def hmcore_cmds(): @click.option("--crypto-key", required=False, type=str) @click.option( "--db-backend", - type=click.Choice(['redis', 'json'], case_sensitive=False), + type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, json." + help="Select the database backend to use. Options: redis, sqlite, json." +) +@click.option( + "--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" +) +@click.option( + "--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" ) @click.option( "--redis-host", default="localhost", - help="Host for Redis (if selected). Default is localhost." + help="[redis] Host for Redis. Default is localhost." ) @click.option( "--redis-port", default=6379, - help="Port for Redis (if selected). Default is 6379." + help="[redis] Port for Redis. Default is 6379." ) def add_client(name, access_key, password, crypto_key, - db_backend, redis_host, redis_port): + db_backend, db_name, db_folder, redis_host, redis_port): + + kwargs = {"backend": db_backend} + if db_backend == "redis": + kwargs["host"] = redis_host + kwargs["port"] = redis_port + else: + kwargs["name"] = db_name + kwargs["subfolder"] = db_folder + key = crypto_key if key: print( @@ -55,10 +72,6 @@ def add_client(name, access_key, password, crypto_key, password = password or os.urandom(16).hex() access_key = access_key or os.urandom(16).hex() - kwargs = {"backend": db_backend} - if db_backend == "redis": - kwargs["host"] = redis_host - kwargs["port"] = redis_port with ClientDatabase(**kwargs) as db: name = name or f"HiveMind-Node-{db.total_clients()}" print(f"Database backend: {db.db.__class__.__name__}") @@ -88,26 +101,37 @@ def add_client(name, access_key, password, crypto_key, @click.argument("node_id", required=False, type=int) @click.option( "--db-backend", - type=click.Choice(['redis', 'json'], case_sensitive=False), + type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, json." + help="Select the database backend to use. Options: redis, sqlite, json." +) +@click.option( + "--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" +) +@click.option( + "--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" ) @click.option( "--redis-host", default="localhost", - help="Host for Redis (if selected). Default is localhost." + help="[redis] Host for Redis. Default is localhost." ) @click.option( "--redis-port", default=6379, - help="Port for Redis (if selected). Default is 6379." + help="[redis] Port for Redis. Default is 6379." ) def allow_msg(msg_type, node_id, - db_backend, redis_host, redis_port): + db_backend, db_name, db_folder, redis_host, redis_port): kwargs = {"backend": db_backend} if db_backend == "redis": kwargs["host"] = redis_host kwargs["port"] = redis_port + else: + kwargs["name"] = db_name + kwargs["subfolder"] = db_folder if not node_id: # list clients and prompt for id using rich @@ -160,26 +184,37 @@ def allow_msg(msg_type, node_id, @click.argument("node_id", required=True, type=int) @click.option( "--db-backend", - type=click.Choice(['redis', 'json'], case_sensitive=False), + type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, json." + help="Select the database backend to use. Options: redis, sqlite, json." +) +@click.option( + "--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" +) +@click.option( + "--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" ) @click.option( "--redis-host", default="localhost", - help="Host for Redis (if selected). Default is localhost." + help="[redis] Host for Redis. Default is localhost." ) @click.option( "--redis-port", default=6379, - help="Port for Redis (if selected). Default is 6379." + help="[redis] Port for Redis. Default is 6379." ) -def delete_client(node_id, +def delete_client(node_id, db_name, db_folder, db_backend, redis_host, redis_port): kwargs = {"backend": db_backend} if db_backend == "redis": kwargs["host"] = redis_host kwargs["port"] = redis_port + else: + kwargs["name"] = db_name + kwargs["subfolder"] = db_folder with ClientDatabase(**kwargs) as db: for x in db: if x["client_id"] == int(node_id): @@ -199,21 +234,29 @@ def delete_client(node_id, @hmcore_cmds.command(help="list clients and credentials", name="list-clients") @click.option( "--db-backend", - type=click.Choice(['redis', 'json'], case_sensitive=False), + type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, json." + help="Select the database backend to use. Options: redis, sqlite, json." +) +@click.option( + "--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" +) +@click.option( + "--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" ) @click.option( "--redis-host", default="localhost", - help="Host for Redis (if selected). Default is localhost." + help="[redis] Host for Redis. Default is localhost." ) @click.option( "--redis-port", default=6379, - help="Port for Redis (if selected). Default is 6379." + help="[redis] Port for Redis. Default is 6379." ) -def list_clients(db_backend, redis_host, redis_port): +def list_clients(db_backend, db_name, db_folder, redis_host, redis_port): console = Console() table = Table(title="HiveMind Credentials:") table.add_column("ID", justify="center") @@ -226,6 +269,9 @@ def list_clients(db_backend, redis_host, redis_port): if db_backend == "redis": kwargs["host"] = redis_host kwargs["port"] = redis_port + else: + kwargs["name"] = db_name + kwargs["subfolder"] = db_folder with ClientDatabase(**kwargs) as db: for x in db: if x["client_id"] != -1: @@ -272,19 +318,27 @@ def list_clients(db_backend, redis_host, redis_port): ) @click.option( "--db-backend", - type=click.Choice(['redis', 'json'], case_sensitive=False), + type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, json." + help="Select the database backend to use. Options: redis, sqlite, json." +) +@click.option( + "--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" +) +@click.option( + "--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" ) @click.option( "--redis-host", default="localhost", - help="Host for Redis (if selected). Default is localhost." + help="[redis] Host for Redis. Default is localhost." ) @click.option( "--redis-port", default=6379, - help="Port for Redis (if selected). Default is 6379." + help="[redis] Port for Redis. Default is 6379." ) def listen( ovos_bus_address: str, @@ -294,7 +348,9 @@ def listen( ssl: bool, cert_dir: str, cert_name: str, - db_backend, redis_host, redis_port + db_backend, + db_name, db_folder, + redis_host, redis_port ): from hivemind_core.service import HiveMindService @@ -315,6 +371,9 @@ def listen( if db_backend == "redis": kwargs["host"] = redis_host kwargs["port"] = redis_port + else: + kwargs["name"] = db_name + kwargs["subfolder"] = db_folder service = HiveMindService( ovos_bus_config=ovos_bus_config, @@ -329,26 +388,37 @@ def listen( @click.argument("node_id", required=False, type=int) @click.option( "--db-backend", - type=click.Choice(['redis', 'json'], case_sensitive=False), + type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, json." + help="Select the database backend to use. Options: redis, sqlite, json." +) +@click.option( + "--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" +) +@click.option( + "--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" ) @click.option( "--redis-host", default="localhost", - help="Host for Redis (if selected). Default is localhost." + help="[redis] Host for Redis. Default is localhost." ) @click.option( "--redis-port", default=6379, - help="Port for Redis (if selected). Default is 6379." + help="[redis] Port for Redis. Default is 6379." ) def blacklist_skill(skill_id, node_id, - db_backend, redis_host, redis_port): + db_backend, db_name, db_folder, redis_host, redis_port): kwargs = {"backend": db_backend} if db_backend == "redis": kwargs["host"] = redis_host kwargs["port"] = redis_port + else: + kwargs["name"] = db_name + kwargs["subfolder"] = db_folder if not node_id: # list clients and prompt for id using rich @@ -401,26 +471,37 @@ def blacklist_skill(skill_id, node_id, @click.argument("node_id", required=False, type=int) @click.option( "--db-backend", - type=click.Choice(['redis', 'json'], case_sensitive=False), + type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, json." + help="Select the database backend to use. Options: redis, sqlite, json." +) +@click.option( + "--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" +) +@click.option( + "--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" ) @click.option( "--redis-host", default="localhost", - help="Host for Redis (if selected). Default is localhost." + help="[redis] Host for Redis. Default is localhost." ) @click.option( "--redis-port", default=6379, - help="Port for Redis (if selected). Default is 6379." + help="[redis] Port for Redis. Default is 6379." ) def unblacklist_skill(skill_id, node_id, - db_backend, redis_host, redis_port): + db_backend,db_name, db_folder, redis_host, redis_port): kwargs = {"backend": db_backend} if db_backend == "redis": kwargs["host"] = redis_host kwargs["port"] = redis_port + else: + kwargs["name"] = db_name + kwargs["subfolder"] = db_folder if not node_id: # list clients and prompt for id using rich @@ -472,26 +553,37 @@ def unblacklist_skill(skill_id, node_id, @click.argument("node_id", required=False, type=int) @click.option( "--db-backend", - type=click.Choice(['redis', 'json'], case_sensitive=False), + type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, json." + help="Select the database backend to use. Options: redis, sqlite, json." +) +@click.option( + "--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" +) +@click.option( + "--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" ) @click.option( "--redis-host", default="localhost", - help="Host for Redis (if selected). Default is localhost." + help="[redis] Host for Redis. Default is localhost." ) @click.option( "--redis-port", default=6379, - help="Port for Redis (if selected). Default is 6379." + help="[redis] Port for Redis. Default is 6379." ) def blacklist_intent(intent_id, node_id, - db_backend, redis_host, redis_port): + db_backend,db_name, db_folder, redis_host, redis_port): kwargs = {"backend": db_backend} if db_backend == "redis": kwargs["host"] = redis_host kwargs["port"] = redis_port + else: + kwargs["name"] = db_name + kwargs["subfolder"] = db_folder if not node_id: # list clients and prompt for id using rich @@ -543,26 +635,37 @@ def blacklist_intent(intent_id, node_id, @click.argument("node_id", required=False, type=int) @click.option( "--db-backend", - type=click.Choice(['redis', 'json'], case_sensitive=False), + type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, json." + help="Select the database backend to use. Options: redis, sqlite, json." +) +@click.option( + "--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" +) +@click.option( + "--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" ) @click.option( "--redis-host", default="localhost", - help="Host for Redis (if selected). Default is localhost." + help="[redis] Host for Redis. Default is localhost." ) @click.option( "--redis-port", default=6379, - help="Port for Redis (if selected). Default is 6379." + help="[redis] Port for Redis. Default is 6379." ) def unblacklist_intent(intent_id, node_id, - db_backend, redis_host, redis_port): + db_backend,db_name, db_folder, redis_host, redis_port): kwargs = {"backend": db_backend} if db_backend == "redis": kwargs["host"] = redis_host kwargs["port"] = redis_port + else: + kwargs["name"] = db_name + kwargs["subfolder"] = db_folder if not node_id: # list clients and prompt for id using rich From e2b026e17ec9c6402d8f6a39627cb2e2d8b07e76 Mon Sep 17 00:00:00 2001 From: miro Date: Fri, 20 Dec 2024 14:12:36 +0000 Subject: [PATCH 04/12] @coderabbitai review --- hivemind_core/scripts.py | 384 ++++++++++---------------------------- hivemind_core/service.py | 6 +- test/unittests/test_db.py | 5 + 3 files changed, 109 insertions(+), 286 deletions(-) diff --git a/hivemind_core/scripts.py b/hivemind_core/scripts.py index 95f63ed..72ad2f5 100644 --- a/hivemind_core/scripts.py +++ b/hivemind_core/scripts.py @@ -9,6 +9,23 @@ from hivemind_core.database import ClientDatabase +def get_db_kwargs(db_backend: str, db_name: str, db_folder: str, + redis_host: str, redis_port: int) -> dict: + """Get database configuration kwargs based on backend type.""" + kwargs = {"backend": db_backend} + if db_backend == "redis": + kwargs.update({ + "host": redis_host, + "port": redis_port + }) + else: + kwargs.update({ + "name": db_name, + "subfolder": db_folder + }) + return kwargs + + @click.group() def hmcore_cmds(): pass @@ -19,40 +36,17 @@ def hmcore_cmds(): @click.option("--access-key", required=False, type=str) @click.option("--password", required=False, type=str) @click.option("--crypto-key", required=False, type=str) -@click.option( - "--db-backend", - type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), - default='json', - help="Select the database backend to use. Options: redis, sqlite, json." -) -@click.option( - "--db-name", type=str, default="clients", - help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" -) -@click.option( - "--db-folder", type=str, default="hivemind-core", - help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" -) -@click.option( - "--redis-host", - default="localhost", - help="[redis] Host for Redis. Default is localhost." -) -@click.option( - "--redis-port", - default=6379, - help="[redis] Port for Redis. Default is 6379." -) +@click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', + help="Select the database backend to use. Options: redis, sqlite, json.") +@click.option("--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") +@click.option("--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") +@click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") +@click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") def add_client(name, access_key, password, crypto_key, db_backend, db_name, db_folder, redis_host, redis_port): - - kwargs = {"backend": db_backend} - if db_backend == "redis": - kwargs["host"] = redis_host - kwargs["port"] = redis_port - else: - kwargs["name"] = db_name - kwargs["subfolder"] = db_folder + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) key = crypto_key if key: @@ -99,39 +93,17 @@ def add_client(name, access_key, password, crypto_key, @hmcore_cmds.command(help="allow message types sent from a client", name="allow-msg") @click.argument("msg_type", required=True, type=str) @click.argument("node_id", required=False, type=int) -@click.option( - "--db-backend", - type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), - default='json', - help="Select the database backend to use. Options: redis, sqlite, json." -) -@click.option( - "--db-name", type=str, default="clients", - help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" -) -@click.option( - "--db-folder", type=str, default="hivemind-core", - help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" -) -@click.option( - "--redis-host", - default="localhost", - help="[redis] Host for Redis. Default is localhost." -) -@click.option( - "--redis-port", - default=6379, - help="[redis] Port for Redis. Default is 6379." -) +@click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', + help="Select the database backend to use. Options: redis, sqlite, json." ) +@click.option("--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") +@click.option("--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") +@click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") +@click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") def allow_msg(msg_type, node_id, db_backend, db_name, db_folder, redis_host, redis_port): - kwargs = {"backend": db_backend} - if db_backend == "redis": - kwargs["host"] = redis_host - kwargs["port"] = redis_port - else: - kwargs["name"] = db_name - kwargs["subfolder"] = db_folder + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) if not node_id: # list clients and prompt for id using rich @@ -182,39 +154,17 @@ def allow_msg(msg_type, node_id, help="remove credentials for a client (numeric unique ID)", name="delete-client" ) @click.argument("node_id", required=True, type=int) -@click.option( - "--db-backend", - type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), - default='json', - help="Select the database backend to use. Options: redis, sqlite, json." -) -@click.option( - "--db-name", type=str, default="clients", - help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" -) -@click.option( - "--db-folder", type=str, default="hivemind-core", - help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" -) -@click.option( - "--redis-host", - default="localhost", - help="[redis] Host for Redis. Default is localhost." -) -@click.option( - "--redis-port", - default=6379, - help="[redis] Port for Redis. Default is 6379." -) +@click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', + help="Select the database backend to use. Options: redis, sqlite, json." ) +@click.option("--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") +@click.option("--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") +@click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") +@click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") def delete_client(node_id, db_name, db_folder, db_backend, redis_host, redis_port): - kwargs = {"backend": db_backend} - if db_backend == "redis": - kwargs["host"] = redis_host - kwargs["port"] = redis_port - else: - kwargs["name"] = db_name - kwargs["subfolder"] = db_folder + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) with ClientDatabase(**kwargs) as db: for x in db: if x["client_id"] == int(node_id): @@ -232,30 +182,14 @@ def delete_client(node_id, db_name, db_folder, @hmcore_cmds.command(help="list clients and credentials", name="list-clients") -@click.option( - "--db-backend", - type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), - default='json', - help="Select the database backend to use. Options: redis, sqlite, json." -) -@click.option( - "--db-name", type=str, default="clients", - help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" -) -@click.option( - "--db-folder", type=str, default="hivemind-core", - help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" -) -@click.option( - "--redis-host", - default="localhost", - help="[redis] Host for Redis. Default is localhost." -) -@click.option( - "--redis-port", - default=6379, - help="[redis] Port for Redis. Default is 6379." -) +@click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', + help="Select the database backend to use. Options: redis, sqlite, json." ) +@click.option("--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") +@click.option("--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") +@click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") +@click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") def list_clients(db_backend, db_name, db_folder, redis_host, redis_port): console = Console() table = Table(title="HiveMind Credentials:") @@ -265,13 +199,7 @@ def list_clients(db_backend, db_name, db_folder, redis_host, redis_port): table.add_column("Password", justify="center") table.add_column("Crypto Key", justify="center") - kwargs = {"backend": db_backend} - if db_backend == "redis": - kwargs["host"] = redis_host - kwargs["port"] = redis_port - else: - kwargs["name"] = db_name - kwargs["subfolder"] = db_folder + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) with ClientDatabase(**kwargs) as db: for x in db: if x["client_id"] != -1: @@ -316,30 +244,14 @@ def list_clients(db_backend, db_name, db_folder, redis_host, redis_port): type=str, default="hivemind", ) -@click.option( - "--db-backend", - type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), - default='json', - help="Select the database backend to use. Options: redis, sqlite, json." -) -@click.option( - "--db-name", type=str, default="clients", - help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" -) -@click.option( - "--db-folder", type=str, default="hivemind-core", - help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" -) -@click.option( - "--redis-host", - default="localhost", - help="[redis] Host for Redis. Default is localhost." -) -@click.option( - "--redis-port", - default=6379, - help="[redis] Port for Redis. Default is 6379." -) +@click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', + help="Select the database backend to use. Options: redis, sqlite, json." ) +@click.option("--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") +@click.option("--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") +@click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") +@click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") def listen( ovos_bus_address: str, ovos_bus_port: int, @@ -367,13 +279,7 @@ def listen( "cert_name": cert_name, } - kwargs = {"backend": db_backend} - if db_backend == "redis": - kwargs["host"] = redis_host - kwargs["port"] = redis_port - else: - kwargs["name"] = db_name - kwargs["subfolder"] = db_folder + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) service = HiveMindService( ovos_bus_config=ovos_bus_config, @@ -386,39 +292,17 @@ def listen( @hmcore_cmds.command(help="blacklist skills from being triggered by a client", name="blacklist-skill") @click.argument("skill_id", required=True, type=str) @click.argument("node_id", required=False, type=int) -@click.option( - "--db-backend", - type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), - default='json', - help="Select the database backend to use. Options: redis, sqlite, json." -) -@click.option( - "--db-name", type=str, default="clients", - help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" -) -@click.option( - "--db-folder", type=str, default="hivemind-core", - help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" -) -@click.option( - "--redis-host", - default="localhost", - help="[redis] Host for Redis. Default is localhost." -) -@click.option( - "--redis-port", - default=6379, - help="[redis] Port for Redis. Default is 6379." -) +@click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', + help="Select the database backend to use. Options: redis, sqlite, json." ) +@click.option("--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") +@click.option("--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") +@click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") +@click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") def blacklist_skill(skill_id, node_id, db_backend, db_name, db_folder, redis_host, redis_port): - kwargs = {"backend": db_backend} - if db_backend == "redis": - kwargs["host"] = redis_host - kwargs["port"] = redis_port - else: - kwargs["name"] = db_name - kwargs["subfolder"] = db_folder + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) if not node_id: # list clients and prompt for id using rich @@ -469,39 +353,17 @@ def blacklist_skill(skill_id, node_id, @hmcore_cmds.command(help="remove skills from a client blacklist", name="unblacklist-skill") @click.argument("skill_id", required=True, type=str) @click.argument("node_id", required=False, type=int) -@click.option( - "--db-backend", - type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), - default='json', - help="Select the database backend to use. Options: redis, sqlite, json." -) -@click.option( - "--db-name", type=str, default="clients", - help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" -) -@click.option( - "--db-folder", type=str, default="hivemind-core", - help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" -) -@click.option( - "--redis-host", - default="localhost", - help="[redis] Host for Redis. Default is localhost." -) -@click.option( - "--redis-port", - default=6379, - help="[redis] Port for Redis. Default is 6379." -) +@click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', + help="Select the database backend to use. Options: redis, sqlite, json." ) +@click.option("--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") +@click.option("--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") +@click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") +@click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") def unblacklist_skill(skill_id, node_id, - db_backend,db_name, db_folder, redis_host, redis_port): - kwargs = {"backend": db_backend} - if db_backend == "redis": - kwargs["host"] = redis_host - kwargs["port"] = redis_port - else: - kwargs["name"] = db_name - kwargs["subfolder"] = db_folder + db_backend, db_name, db_folder, redis_host, redis_port): + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) if not node_id: # list clients and prompt for id using rich @@ -551,39 +413,17 @@ def unblacklist_skill(skill_id, node_id, @hmcore_cmds.command(help="blacklist intents from being triggered by a client", name="blacklist-intent") @click.argument("intent_id", required=True, type=str) @click.argument("node_id", required=False, type=int) -@click.option( - "--db-backend", - type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), - default='json', - help="Select the database backend to use. Options: redis, sqlite, json." -) -@click.option( - "--db-name", type=str, default="clients", - help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" -) -@click.option( - "--db-folder", type=str, default="hivemind-core", - help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" -) -@click.option( - "--redis-host", - default="localhost", - help="[redis] Host for Redis. Default is localhost." -) -@click.option( - "--redis-port", - default=6379, - help="[redis] Port for Redis. Default is 6379." -) +@click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', + help="Select the database backend to use. Options: redis, sqlite, json." ) +@click.option("--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") +@click.option("--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") +@click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") +@click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") def blacklist_intent(intent_id, node_id, - db_backend,db_name, db_folder, redis_host, redis_port): - kwargs = {"backend": db_backend} - if db_backend == "redis": - kwargs["host"] = redis_host - kwargs["port"] = redis_port - else: - kwargs["name"] = db_name - kwargs["subfolder"] = db_folder + db_backend, db_name, db_folder, redis_host, redis_port): + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) if not node_id: # list clients and prompt for id using rich @@ -633,39 +473,17 @@ def blacklist_intent(intent_id, node_id, @hmcore_cmds.command(help="remove intents from a client blacklist", name="unblacklist-intent") @click.argument("intent_id", required=True, type=str) @click.argument("node_id", required=False, type=int) -@click.option( - "--db-backend", - type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), - default='json', - help="Select the database backend to use. Options: redis, sqlite, json." -) -@click.option( - "--db-name", type=str, default="clients", - help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}" -) -@click.option( - "--db-folder", type=str, default="hivemind-core", - help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}" -) -@click.option( - "--redis-host", - default="localhost", - help="[redis] Host for Redis. Default is localhost." -) -@click.option( - "--redis-port", - default=6379, - help="[redis] Port for Redis. Default is 6379." -) +@click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', + help="Select the database backend to use. Options: redis, sqlite, json." ) +@click.option("--db-name", type=str, default="clients", + help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") +@click.option("--db-folder", type=str, default="hivemind-core", + help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") +@click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") +@click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") def unblacklist_intent(intent_id, node_id, - db_backend,db_name, db_folder, redis_host, redis_port): - kwargs = {"backend": db_backend} - if db_backend == "redis": - kwargs["host"] = redis_host - kwargs["port"] = redis_port - else: - kwargs["name"] = db_name - kwargs["subfolder"] = db_folder + db_backend, db_name, db_folder, redis_host, redis_port): + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) if not node_id: # list clients and prompt for id using rich diff --git a/hivemind_core/service.py b/hivemind_core/service.py index 962c737..cc5a9bf 100644 --- a/hivemind_core/service.py +++ b/hivemind_core/service.py @@ -136,11 +136,11 @@ def open(self): loop=self.protocol.loop, ) if self.db is None: + # should never happen, but double check! LOG.error("Database connection not initialized. Please ensure database configuration is correct.") - LOG.debug(f"Client {client.peer} connection attempt failed due to missing database connection") - self.protocol.handle_invalid_key_connected(self.client) + LOG.exception(f"Client {self.client.peer} connection attempt failed due to missing database connection") self.close() - return + raise RuntimeError("Database was not initialized!") # let it propagate, this is developer error most likely user = self.db.get_client_by_api_key(key) if not user: diff --git a/test/unittests/test_db.py b/test/unittests/test_db.py index 88412f1..12ac3e6 100644 --- a/test/unittests/test_db.py +++ b/test/unittests/test_db.py @@ -1,4 +1,5 @@ import json +import os import unittest from unittest.mock import patch, MagicMock @@ -71,6 +72,10 @@ class TestJsonDB(unittest.TestCase): def setUp(self): self.db = JsonDB(name=".hivemind-test") + def tearDown(self): + if os.path.exists(self.db._db.path): + os.remove(self.db._db.path) + def test_add_item(self): client_data = { "client_id": 1, From 6c4f534458c0e078c924122b08ebe7d4cd33f555 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 20 Dec 2024 14:45:58 +0000 Subject: [PATCH 05/12] Update README.md --- README.md | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 35af18e..c58cfbd 100644 --- a/README.md +++ b/README.md @@ -258,9 +258,16 @@ HiveMind-Core supports multiple database backends to store client credentials an ## 🧩 HiveMind Ecosystem -### Core Components +### Minds + +You must run at least one of these - **HiveMind Core** (this repository): The central hub for managing connections and routing messages between devices. + *text* input only +- [Hivemind Listener](https://github.com/JarbasHiveMind/HiveMind-listener) - an extension of `hivemind-core` for + streaming *audio* from satellites **<- you likely want this one** +- [Persona](https://github.com/JarbasHiveMind/HiveMind-persona) - run + a [persona](https://github.com/OpenVoiceOS/ovos-persona) (eg. LLM). *text* input only ### Client Libraries @@ -269,21 +276,36 @@ HiveMind-Core supports multiple database backends to store client credentials an ### Terminals -- [Remote CLI](https://github.com/OpenJarbas/HiveMind-cli) (**Recommended Starting Point**) -- [Voice Satellite](https://github.com/OpenJarbas/HiveMind-voice-sat) -- [Flask Chatroom](https://github.com/JarbasHiveMind/HiveMind-flask-template) -- [Web Chat](https://github.com/OpenJarbas/HiveMind-webchat) +- [Voice Satellite](https://github.com/OpenJarbas/HiveMind-voice-sat) (standalone OVOS *local* audio stack) +- [Voice Relay](https://github.com/JarbasHiveMind/HiveMind-voice-relay) (lightweight audio satellite, STT/TTS + processed *server* side, **requires** `hivemind-listener`) +- [Mic Satellite](https://github.com/JarbasHiveMind/hivemind-mic-satellite) (only VAD runs on device, audio streamed + and fully processed *server* side, **requires** `hivemind-listener`) +- [Web Chat](https://github.com/OpenJarbas/HiveMind-webchat) (*client-side* browser hivemind connection) +- [Flask Chatroom](https://github.com/JarbasHiveMind/HiveMind-flask-template) (**boilerplate template** - *server-side* + hivemind connection) ### Bridges - [Mattermost Bridge](https://github.com/OpenJarbas/HiveMind_mattermost_bridge) -- [HackChat Bridge](https://github.com/OpenJarbas/HiveMind-HackChatBridge) -- [Twitch Bridge](https://github.com/OpenJarbas/HiveMind-twitch-bridge) - [DeltaChat Bridge](https://github.com/JarbasHiveMind/HiveMind-deltachat-bridge) -### Minds +--- -- [NodeRed Integration](https://github.com/OpenJarbas/HiveMind-NodeRed) +## Voice Satellite Comparison + +| Feature | **HiveMind Voice Satellite** | **HiveMind Voice Relay** | **HiveMind Microphone Satellite** | +|------------------------------------|--------------------------------------------|-------------------------------------------------------------------------------------------|--------------------------------------------------------------| +| **Use Case** | Full voice satellite with local processing | Offloads voice processing to **HiveMind Listener**, better suited for secure environments | Super lightweight satellite for fully remote audio streaming | +| **Audio Processing** | Local (STT, TTS, Wake Word, VAD) | STT and TTS handled on **HiveMind Listener** | Wake Word, STT and TTS handled on **HiveMind Listener** | +| **Server Requirements** | Works with **all** mind implementations | Requires **hivemind-listener** | Requires **hivemind-listener** | +| **Voice Activity Detection (VAD)** | Handled locally | Handled locally | Handled locally | +| **Wake Word Detection** | Handled locally | Handled locally | Handled by **HiveMind Listener** | +| **STT Processing** | Handled locally | Handled by **HiveMind Listener** | Handled by **HiveMind Listener** | +| **TTS Processing** | Handled locally | Handled by **HiveMind Listener** | Handled by **HiveMind Listener** | +| **Media Playback Support** | Yes, with supported plugins | Yes, with supported plugins | No | +| **PHAL plugins** | Supported | Supported | Unsupported | +| **Transformer plugins** | Supported | Supported | Unsupported | --- From 255a8656521a2677aa70318732c731965fb5b971 Mon Sep 17 00:00:00 2001 From: miro Date: Fri, 20 Dec 2024 15:12:31 +0000 Subject: [PATCH 06/12] improve redis search --- hivemind_core/database.py | 84 +++++++++++++-------------------------- test/unittests/test_db.py | 46 +++++++++++++++++++++ 2 files changed, 74 insertions(+), 56 deletions(-) diff --git a/hivemind_core/database.py b/hivemind_core/database.py index dcef3ef..b4495c9 100644 --- a/hivemind_core/database.py +++ b/hivemind_core/database.py @@ -67,6 +67,10 @@ def __post_init__(self): """ Initializes the allowed types for the Client instance if not provided. """ + if not isinstance(self.client_id, int): + raise ValueError("client_id should be an integer") + if not isinstance(self.is_admin, bool): + raise ValueError("is_admin should be a boolean") self.allowed_types = self.allowed_types or ["recognizer_loop:utterance", "recognizer_loop:record_begin", "recognizer_loop:record_end", @@ -185,7 +189,6 @@ def add_item(self, client: Client) -> bool: """ pass - @abc.abstractmethod def delete_item(self, client: Client) -> bool: """ Delete a client from the database. @@ -196,7 +199,9 @@ def delete_item(self, client: Client) -> bool: Returns: True if the deletion was successful, False otherwise. """ - pass + # leave the deleted entry in db, do not allow reuse of client_id ! + client = Client(client_id=client.client_id, api_key="revoked") + return self.update_item(client) def update_item(self, client: Client) -> bool: """ @@ -287,21 +292,6 @@ def add_item(self, client: Client) -> bool: self._db[client.client_id] = client.__dict__ return True - def delete_item(self, client: Client) -> bool: - """ - Delete a client from the JSON database. - - Args: - client: The client to be deleted. - - Returns: - True if the deletion was successful, False otherwise. - """ - if client.client_id in self._db: - self._db.pop(client.client_id) - return True - return False - def search_by_value(self, key: str, val: Union[str, bool, int, float]) -> List[Client]: """ Search for clients by a specific key-value pair in the JSON database. @@ -435,24 +425,6 @@ def add_item(self, client: Client) -> bool: LOG.error(f"Failed to add client to SQLite: {e}") return False - def delete_item(self, client: Client) -> bool: - """ - Delete a client from the SQLite database. - - Args: - client: The client to be deleted. - - Returns: - True if the deletion was successful, False otherwise. - """ - try: - with self.conn: - self.conn.execute("DELETE FROM clients WHERE client_id = ?", (client.client_id,)) - return True - except sqlite3.Error as e: - LOG.error(f"Failed to delete client from SQLite: {e}") - return False - def search_by_value(self, key: str, val: Union[str, bool, int, float]) -> List[Client]: """ Search for clients by a specific key-value pair in the SQLite database. @@ -549,31 +521,16 @@ def add_item(self, client: Client) -> bool: """ item_key = f"client:{client.client_id}" serialized_data: str = client.serialize() - try: # Store data in Redis self.redis.set(item_key, serialized_data) - return True - except Exception as e: - LOG.error(f"Failed to add client to Redis/RediSearch: {e}") - return False - def delete_item(self, client: Client) -> bool: - """ - Delete a client from Redis and RediSearch. - - Args: - client: The client to be deleted. - - Returns: - True if the deletion was successful, False otherwise. - """ - item_key = f"client:{client.client_id}" - try: - self.redis.delete(item_key) + # Maintain indices for common search fields + self.redis.sadd(f"client:index:name:{client.name}", client.client_id) + self.redis.sadd(f"client:index:api_key:{client.api_key}", client.client_id) return True except Exception as e: - LOG.error(f"Failed to delete client from Redis: {e}") + LOG.error(f"Failed to add client to Redis/RediSearch: {e}") return False def search_by_value(self, key: str, val: Union[str, bool, int, float]) -> List[Client]: @@ -587,8 +544,18 @@ def search_by_value(self, key: str, val: Union[str, bool, int, float]) -> List[C Returns: A list of clients that match the search criteria. """ + # Use index if available + if key in ['name', 'api_key']: + client_ids = self.redis.smembers(f"client:index:{key}:{val}") + res = [cast2client(self.redis.get(f"client:{cid}")) + for cid in client_ids] + res = [c for c in res if c.api_key != "revoked"] + return res + res = [] for client_id in self.redis.scan_iter(f"client:*"): + if client_id.startswith("client:index:"): + continue client_data = self.redis.get(client_id) client = cast2client(client_data) if hasattr(client, key) and getattr(client, key) == val: @@ -602,7 +569,7 @@ def __len__(self) -> int: Returns: The number of clients in the database. """ - return len(self.redis.keys("client:*")) + return int(len(self.redis.keys("client:*")) / 3) # because of index entries for name/key fastsearch def __iter__(self) -> Iterable['Client']: """ @@ -612,7 +579,12 @@ def __iter__(self) -> Iterable['Client']: An iterator over the clients in the database. """ for client_id in self.redis.scan_iter(f"client:*"): - yield cast2client(self.redis.get(client_id)) + if client_id.startswith("client:index:"): + continue + try: + yield cast2client(self.redis.get(client_id)) + except Exception as e: + LOG.error(f"Failed to get client '{client_id}' : {e}") class ClientDatabase: diff --git a/test/unittests/test_db.py b/test/unittests/test_db.py index 12ac3e6..df95b41 100644 --- a/test/unittests/test_db.py +++ b/test/unittests/test_db.py @@ -183,5 +183,51 @@ def test_get_clients_by_name(self): self.assertEqual(clients[0].name, "Test Client") +class TestClientNegativeCases(unittest.TestCase): + + def test_missing_required_fields(self): + # Missing the "client_id" field, which is required by the Client dataclass + client_data = { + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": False + } + with self.assertRaises(TypeError): + Client(**client_data) + + def test_invalid_field_type_for_client_id(self): + # Providing a string instead of an integer for "client_id" + client_data = { + "client_id": "invalid_id", + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": False + } + with self.assertRaises(ValueError): + # If needed, adjust logic in your code to raise ValueError instead of TypeError + Client(**client_data) + + def test_invalid_field_type_for_is_admin(self): + # Providing a string instead of a boolean for "is_admin" + client_data = { + "client_id": 1, + "api_key": "test_api_key", + "name": "Test Client", + "description": "A test client", + "is_admin": "not_boolean" + } + with self.assertRaises(ValueError): + # If needed, adjust logic in your code to raise ValueError instead of TypeError + Client(**client_data) + + def test_deserialize_with_incorrect_json_structure(self): + # Passing an invalid JSON string missing required fields + invalid_json_str = '{"client_id": 1}' + with self.assertRaises(TypeError): + # Or another appropriate exception if your parsing logic differs + Client.deserialize(invalid_json_str) + if __name__ == '__main__': unittest.main() From 2585235305e01fc09629ca2d8e503052be82819d Mon Sep 17 00:00:00 2001 From: miro Date: Fri, 20 Dec 2024 15:26:21 +0000 Subject: [PATCH 07/12] ensure typing in sqlite --- hivemind_core/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hivemind_core/database.py b/hivemind_core/database.py index b4495c9..cbd3399 100644 --- a/hivemind_core/database.py +++ b/hivemind_core/database.py @@ -474,11 +474,11 @@ def commit(self) -> bool: def _row_to_client(row: sqlite3.Row) -> Client: """Convert a database row to a Client instance.""" return Client( - client_id=row["client_id"], + client_id=int(row["client_id"]), api_key=row["api_key"], name=row["name"], description=row["description"], - is_admin=row["is_admin"], + is_admin=row["is_admin"] or False, last_seen=row["last_seen"], intent_blacklist=json.loads(row["intent_blacklist"] or "[]"), skill_blacklist=json.loads(row["skill_blacklist"] or "[]"), From b5331f4ee56375015899daedb6bdf81e225c049c Mon Sep 17 00:00:00 2001 From: miro Date: Fri, 20 Dec 2024 15:41:44 +0000 Subject: [PATCH 08/12] fix db paths --- README.md | 10 +++++----- hivemind_core/database.py | 17 +++++++++-------- test/unittests/test_db.py | 1 + 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c58cfbd..8d55034 100644 --- a/README.md +++ b/README.md @@ -228,11 +228,11 @@ Kubernetes), use the `--ovos_bus_address` and `--ovos_bus_port` options to speci HiveMind-Core supports multiple database backends to store client credentials and settings. Each has its own use case: -| Backend | Use Case | Default Location | Command Line options | -|--------------------|------------------------------------------------|---------------------------------------|----------------------------------------------------| -| **JSON** (default) | Simple, file-based setup for local use | `~/.cache/hivemind-core/clients.json` | Configurable via `--db-name` and `--db-folder` | -| **SQLite** | Lightweight relational DB for single instances | `~/.cache/hivemind-core/clients.db` | Configurable via `--db-name` and `--db-folder` | -| **Redis** | Distributed, high-performance environments | `localhost:6379` | Configurable via `--redis-host` and `--redis-port` | +| Backend | Use Case | Default Location | Command Line options | +|--------------------|------------------------------------------------|---------------------------------------------|----------------------------------------------------| +| **JSON** (default) | Simple, file-based setup for local use | `~/.local/share/hivemind-core/clients.json` | Configurable via `--db-name` and `--db-folder` | +| **SQLite** | Lightweight relational DB for single instances | `~/.local/share/hivemind-core/clients.db` | Configurable via `--db-name` and `--db-folder` | +| **Redis** | Distributed, high-performance environments | `localhost:6379` | Configurable via `--redis-host` and `--redis-port` | **How to Choose?** diff --git a/hivemind_core/database.py b/hivemind_core/database.py index cbd3399..e61ceac 100644 --- a/hivemind_core/database.py +++ b/hivemind_core/database.py @@ -277,7 +277,8 @@ class JsonDB(AbstractDB): """Database implementation using JSON files.""" def __init__(self, name="clients", subfolder="hivemind-core"): - self._db: Dict[int, ClientDict] = JsonStorageXDG(name, subfolder=subfolder) + self._db: Dict[int, ClientDict] = JsonStorageXDG(name, subfolder=subfolder, xdg_folder=xdg_data_home()) + LOG.debug(f"json database path: {self._db.path}") def add_item(self, client: Client) -> bool: """ @@ -355,13 +356,11 @@ class SQLiteDB(AbstractDB): def __init__(self, name="clients", subfolder="hivemind-core"): """ Initialize the SQLiteDB connection. - - Args: - db_path: Path to the SQLite database file. Default is in-memory. """ if sqlite3 is None: raise ImportError("pip install sqlite3") db_path = os.path.join(xdg_data_home(), subfolder, name + ".db") + LOG.debug(f"sqlite database path: {db_path}") os.makedirs(os.path.dirname(db_path), exist_ok=True) self.conn = sqlite3.connect(db_path) @@ -371,19 +370,21 @@ def __init__(self, name="clients", subfolder="hivemind-core"): def _initialize_database(self): """Initialize the database schema.""" with self.conn: + # crypto key is always 16 chars + # name description and api_key shouldnt be allowed to go over 255 self.conn.execute(""" CREATE TABLE IF NOT EXISTS clients ( client_id INTEGER PRIMARY KEY, - api_key TEXT NOT NULL, - name TEXT, - description TEXT, + api_key VARCHAR(255) NOT NULL, + name VARCHAR(255), + description VARCHAR(255), is_admin BOOLEAN DEFAULT FALSE, last_seen REAL DEFAULT -1, intent_blacklist TEXT, skill_blacklist TEXT, message_blacklist TEXT, allowed_types TEXT, - crypto_key TEXT, + crypto_key VARCHAR(16), password TEXT, can_broadcast BOOLEAN DEFAULT TRUE, can_escalate BOOLEAN DEFAULT TRUE, diff --git a/test/unittests/test_db.py b/test/unittests/test_db.py index df95b41..bbbdb2e 100644 --- a/test/unittests/test_db.py +++ b/test/unittests/test_db.py @@ -229,5 +229,6 @@ def test_deserialize_with_incorrect_json_structure(self): # Or another appropriate exception if your parsing logic differs Client.deserialize(invalid_json_str) + if __name__ == '__main__': unittest.main() From d5ffc905ce4c03a66e9dfb3e24ddb766b10fa4d5 Mon Sep 17 00:00:00 2001 From: miro Date: Fri, 20 Dec 2024 19:51:40 +0000 Subject: [PATCH 09/12] fix --- hivemind_core/protocol.py | 5 +++-- hivemind_core/service.py | 15 +++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hivemind_core/protocol.py b/hivemind_core/protocol.py index f1bc09a..78bd172 100644 --- a/hivemind_core/protocol.py +++ b/hivemind_core/protocol.py @@ -758,8 +758,9 @@ def _update_blacklist(self, message: Message, client: HiveMindClientConnection): # update blacklist from db, to account for changes without requiring a restart user = self.db.get_client_by_api_key(client.key) - client.skill_blacklist = user.skill_blacklist - client.intent_blacklist = user.intent_blacklist + client.skill_blacklist = user.skill_blacklist or [] + client.intent_blacklist = user.intent_blacklist or [] + client.msg_blacklist = user.message_blacklist or [] # inject client specific blacklist into session if "blacklisted_skills" not in message.context["session"]: diff --git a/hivemind_core/service.py b/hivemind_core/service.py index cc5a9bf..89c1c46 100644 --- a/hivemind_core/service.py +++ b/hivemind_core/service.py @@ -99,7 +99,6 @@ def on_stopping(): class MessageBusEventHandler(WebSocketHandler): protocol: Optional[HiveMindListenerProtocol] = None - db: Optional[ClientDatabase] = None @staticmethod def decode_auth(auth) -> Tuple[str, str]: @@ -135,14 +134,15 @@ def open(self): handshake=handshake, loop=self.protocol.loop, ) - if self.db is None: + if self.protocol.db is None: # should never happen, but double check! LOG.error("Database connection not initialized. Please ensure database configuration is correct.") LOG.exception(f"Client {self.client.peer} connection attempt failed due to missing database connection") self.close() raise RuntimeError("Database was not initialized!") # let it propagate, this is developer error most likely - user = self.db.get_client_by_api_key(key) + user = self.protocol.db.get_client_by_api_key(key) + if not user: LOG.error("Client provided an invalid api key") self.protocol.handle_invalid_key_connected(self.client) @@ -150,9 +150,9 @@ def open(self): return self.client.crypto_key = user.crypto_key - self.client.msg_blacklist = user.message_blacklist - self.client.skill_blacklist = user.skill_blacklist - self.client.intent_blacklist = user.intent_blacklist + self.client.msg_blacklist = user.message_blacklist or [] + self.client.skill_blacklist = user.skill_blacklist or [] + self.client.intent_blacklist = user.intent_blacklist or [] self.client.allowed_types = user.allowed_types self.client.can_broadcast = user.can_broadcast self.client.can_propagate = user.can_propagate @@ -215,10 +215,9 @@ def __init__( on_error=error_hook, on_stopping=stopping_hook, ) - self.db = db + self.db = db or ClientDatabase() self._proto = protocol self._ws_handler = ws_handler - self._ws_handler.db = db if bus: self.bus = bus else: From 737854121597dc6dfa721f87646d4dffb66bf54e Mon Sep 17 00:00:00 2001 From: miro Date: Fri, 20 Dec 2024 20:22:09 +0000 Subject: [PATCH 10/12] redis password --- hivemind_core/database.py | 6 ++- hivemind_core/scripts.py | 84 ++++++++++++++++++++------------------- 2 files changed, 47 insertions(+), 43 deletions(-) diff --git a/hivemind_core/database.py b/hivemind_core/database.py index e61ceac..4edc98b 100644 --- a/hivemind_core/database.py +++ b/hivemind_core/database.py @@ -496,7 +496,7 @@ def _row_to_client(row: sqlite3.Row) -> Client: class RedisDB(AbstractDB): """Database implementation using Redis with RediSearch support.""" - def __init__(self, host: str = "127.0.0.1", port: int = 6379, redis_db: int = 0): + def __init__(self, host: str = "127.0.0.1", port: int = 6379, password: Optional[str] = None, redis_db: int = 0): """ Initialize the RedisDB connection. @@ -507,7 +507,9 @@ def __init__(self, host: str = "127.0.0.1", port: int = 6379, redis_db: int = 0) """ if redis is None: raise ImportError("pip install redis") - self.redis = redis.StrictRedis(host=host, port=port, db=redis_db, decode_responses=True) + self.redis = redis.StrictRedis(host=host, port=port, db=redis_db, + password=password if password else None, + decode_responses=True) # TODO - support for a proper search index def add_item(self, client: Client) -> bool: diff --git a/hivemind_core/scripts.py b/hivemind_core/scripts.py index 72ad2f5..568f735 100644 --- a/hivemind_core/scripts.py +++ b/hivemind_core/scripts.py @@ -1,4 +1,5 @@ import os +from typing import Optional import click from ovos_utils.xdg_utils import xdg_data_home @@ -10,13 +11,14 @@ def get_db_kwargs(db_backend: str, db_name: str, db_folder: str, - redis_host: str, redis_port: int) -> dict: + redis_host: str, redis_port: int, redis_password: Optional[str]) -> dict: """Get database configuration kwargs based on backend type.""" kwargs = {"backend": db_backend} if db_backend == "redis": kwargs.update({ "host": redis_host, - "port": redis_port + "port": redis_port, + "password": redis_password }) else: kwargs.update({ @@ -44,9 +46,11 @@ def hmcore_cmds(): help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") +@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") def add_client(name, access_key, password, crypto_key, - db_backend, db_name, db_folder, redis_host, redis_port): - kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) + db_backend, db_name, db_folder, + redis_host, redis_port, redis_password): + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) key = crypto_key if key: @@ -94,16 +98,17 @@ def add_client(name, access_key, password, crypto_key, @click.argument("msg_type", required=True, type=str) @click.argument("node_id", required=False, type=int) @click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, sqlite, json." ) + help="Select the database backend to use. Options: redis, sqlite, json.") @click.option("--db-name", type=str, default="clients", help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") @click.option("--db-folder", type=str, default="hivemind-core", help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") +@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") def allow_msg(msg_type, node_id, - db_backend, db_name, db_folder, redis_host, redis_port): - kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) + db_backend, db_name, db_folder, redis_host, redis_port, redis_password): + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) if not node_id: # list clients and prompt for id using rich @@ -155,16 +160,17 @@ def allow_msg(msg_type, node_id, ) @click.argument("node_id", required=True, type=int) @click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, sqlite, json." ) + help="Select the database backend to use. Options: redis, sqlite, json.") @click.option("--db-name", type=str, default="clients", help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") @click.option("--db-folder", type=str, default="hivemind-core", help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") +@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") def delete_client(node_id, db_name, db_folder, - db_backend, redis_host, redis_port): - kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) + db_backend, redis_host, redis_port, redis_password): + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) with ClientDatabase(**kwargs) as db: for x in db: if x["client_id"] == int(node_id): @@ -183,14 +189,16 @@ def delete_client(node_id, db_name, db_folder, @hmcore_cmds.command(help="list clients and credentials", name="list-clients") @click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, sqlite, json." ) + help="Select the database backend to use. Options: redis, sqlite, json.") @click.option("--db-name", type=str, default="clients", help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") @click.option("--db-folder", type=str, default="hivemind-core", help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") -def list_clients(db_backend, db_name, db_folder, redis_host, redis_port): +@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") +def list_clients(db_backend, db_name, db_folder, redis_host, redis_port, redis_password): + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) console = Console() table = Table(title="HiveMind Credentials:") table.add_column("ID", justify="center") @@ -199,7 +207,6 @@ def list_clients(db_backend, db_name, db_folder, redis_host, redis_port): table.add_column("Password", justify="center") table.add_column("Crypto Key", justify="center") - kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) with ClientDatabase(**kwargs) as db: for x in db: if x["client_id"] != -1: @@ -245,25 +252,19 @@ def list_clients(db_backend, db_name, db_folder, redis_host, redis_port): default="hivemind", ) @click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, sqlite, json." ) + help="Select the database backend to use. Options: redis, sqlite, json.") @click.option("--db-name", type=str, default="clients", help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") @click.option("--db-folder", type=str, default="hivemind-core", help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") -def listen( - ovos_bus_address: str, - ovos_bus_port: int, - host: str, - port: int, - ssl: bool, - cert_dir: str, - cert_name: str, - db_backend, - db_name, db_folder, - redis_host, redis_port -): +@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") +def listen(ovos_bus_address: str, ovos_bus_port: int, host: str, port: int, + ssl: bool, cert_dir: str, cert_name: str, + db_backend, db_name, db_folder, + redis_host, redis_port, redis_password): + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) from hivemind_core.service import HiveMindService ovos_bus_config = { @@ -278,9 +279,6 @@ def listen( "cert_dir": cert_dir, "cert_name": cert_name, } - - kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) - service = HiveMindService( ovos_bus_config=ovos_bus_config, websocket_config=websocket_config, @@ -293,16 +291,17 @@ def listen( @click.argument("skill_id", required=True, type=str) @click.argument("node_id", required=False, type=int) @click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, sqlite, json." ) + help="Select the database backend to use. Options: redis, sqlite, json.") @click.option("--db-name", type=str, default="clients", help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") @click.option("--db-folder", type=str, default="hivemind-core", help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") +@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") def blacklist_skill(skill_id, node_id, - db_backend, db_name, db_folder, redis_host, redis_port): - kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) + db_backend, db_name, db_folder, redis_host, redis_port, redis_password): + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) if not node_id: # list clients and prompt for id using rich @@ -354,16 +353,17 @@ def blacklist_skill(skill_id, node_id, @click.argument("skill_id", required=True, type=str) @click.argument("node_id", required=False, type=int) @click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, sqlite, json." ) + help="Select the database backend to use. Options: redis, sqlite, json.") @click.option("--db-name", type=str, default="clients", help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") @click.option("--db-folder", type=str, default="hivemind-core", help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") +@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") def unblacklist_skill(skill_id, node_id, - db_backend, db_name, db_folder, redis_host, redis_port): - kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) + db_backend, db_name, db_folder, redis_host, redis_port, redis_password): + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) if not node_id: # list clients and prompt for id using rich @@ -414,16 +414,17 @@ def unblacklist_skill(skill_id, node_id, @click.argument("intent_id", required=True, type=str) @click.argument("node_id", required=False, type=int) @click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, sqlite, json." ) + help="Select the database backend to use. Options: redis, sqlite, json.") @click.option("--db-name", type=str, default="clients", help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") @click.option("--db-folder", type=str, default="hivemind-core", help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") +@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") def blacklist_intent(intent_id, node_id, - db_backend, db_name, db_folder, redis_host, redis_port): - kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) + db_backend, db_name, db_folder, redis_host, redis_port, redis_password): + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) if not node_id: # list clients and prompt for id using rich @@ -474,16 +475,17 @@ def blacklist_intent(intent_id, node_id, @click.argument("intent_id", required=True, type=str) @click.argument("node_id", required=False, type=int) @click.option("--db-backend", type=click.Choice(['redis', 'json', 'sqlite'], case_sensitive=False), default='json', - help="Select the database backend to use. Options: redis, sqlite, json." ) + help="Select the database backend to use. Options: redis, sqlite, json.") @click.option("--db-name", type=str, default="clients", help="[json/sqlite] The name for the database file. ~/.cache/hivemind-core/{name}") @click.option("--db-folder", type=str, default="hivemind-core", help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") +@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") def unblacklist_intent(intent_id, node_id, - db_backend, db_name, db_folder, redis_host, redis_port): - kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port) + db_backend, db_name, db_folder, redis_host, redis_port, redis_password): + kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) if not node_id: # list clients and prompt for id using rich From 96c10497241e70f12cd8bafb4f1892cf40d8cb7c Mon Sep 17 00:00:00 2001 From: miro Date: Fri, 20 Dec 2024 20:24:12 +0000 Subject: [PATCH 11/12] redis password --- hivemind_core/scripts.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/hivemind_core/scripts.py b/hivemind_core/scripts.py index 568f735..d158d74 100644 --- a/hivemind_core/scripts.py +++ b/hivemind_core/scripts.py @@ -46,7 +46,7 @@ def hmcore_cmds(): help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") -@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") +@click.option("--redis-password", required=False, help="[redis] Password for Redis. Default None") def add_client(name, access_key, password, crypto_key, db_backend, db_name, db_folder, redis_host, redis_port, redis_password): @@ -105,7 +105,7 @@ def add_client(name, access_key, password, crypto_key, help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") -@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") +@click.option("--redis-password", required=False, help="[redis] Password for Redis. Default None") def allow_msg(msg_type, node_id, db_backend, db_name, db_folder, redis_host, redis_port, redis_password): kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) @@ -167,7 +167,7 @@ def allow_msg(msg_type, node_id, help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") -@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") +@click.option("--redis-password", required=False, help="[redis] Password for Redis. Default None") def delete_client(node_id, db_name, db_folder, db_backend, redis_host, redis_port, redis_password): kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) @@ -196,7 +196,7 @@ def delete_client(node_id, db_name, db_folder, help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") -@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") +@click.option("--redis-password", required=False, help="[redis] Password for Redis. Default None") def list_clients(db_backend, db_name, db_folder, redis_host, redis_port, redis_password): kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) console = Console() @@ -259,7 +259,7 @@ def list_clients(db_backend, db_name, db_folder, redis_host, redis_port, redis_p help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") -@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") +@click.option("--redis-password", required=False, help="[redis] Password for Redis. Default None") def listen(ovos_bus_address: str, ovos_bus_port: int, host: str, port: int, ssl: bool, cert_dir: str, cert_name: str, db_backend, db_name, db_folder, @@ -298,7 +298,7 @@ def listen(ovos_bus_address: str, ovos_bus_port: int, host: str, port: int, help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") -@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") +@click.option("--redis-password", required=False, help="[redis] Password for Redis. Default None") def blacklist_skill(skill_id, node_id, db_backend, db_name, db_folder, redis_host, redis_port, redis_password): kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) @@ -360,7 +360,7 @@ def blacklist_skill(skill_id, node_id, help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") -@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") +@click.option("--redis-password", required=False, help="[redis] Password for Redis. Default None") def unblacklist_skill(skill_id, node_id, db_backend, db_name, db_folder, redis_host, redis_port, redis_password): kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) @@ -421,7 +421,7 @@ def unblacklist_skill(skill_id, node_id, help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") -@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") +@click.option("--redis-password", required=False, help="[redis] Password for Redis. Default None") def blacklist_intent(intent_id, node_id, db_backend, db_name, db_folder, redis_host, redis_port, redis_password): kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) @@ -482,7 +482,7 @@ def blacklist_intent(intent_id, node_id, help="[json/sqlite] The subfolder where database files are stored. ~/.cache/{db_folder}}") @click.option("--redis-host", default="localhost", help="[redis] Host for Redis. Default is localhost.") @click.option("--redis-port", default=6379, help="[redis] Port for Redis. Default is 6379.") -@click.option("--redis-password", required=FileNotFoundError, help="[redis] Password for Redis. Default None") +@click.option("--redis-password", required=False, help="[redis] Password for Redis. Default None") def unblacklist_intent(intent_id, node_id, db_backend, db_name, db_folder, redis_host, redis_port, redis_password): kwargs = get_db_kwargs(db_backend, db_name, db_folder, redis_host, redis_port, redis_password) From 6cb0874780d85784e4e4d34d69008874941f8e43 Mon Sep 17 00:00:00 2001 From: miro Date: Fri, 20 Dec 2024 20:25:44 +0000 Subject: [PATCH 12/12] reset version to 0.1.0 --- hivemind_core/version.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hivemind_core/version.py b/hivemind_core/version.py index 67176bc..be3eeb2 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 14 -VERSION_BUILD = 1 -VERSION_ALPHA = 0 +VERSION_MINOR = 1 +VERSION_BUILD = 0 +VERSION_ALPHA = 1 # END_VERSION_BLOCK