diff --git a/src/tribler/core/components.py b/src/tribler/core/components.py index 339922415d..1d26cd217d 100644 --- a/src/tribler/core/components.py +++ b/src/tribler/core/components.py @@ -1,7 +1,7 @@ from __future__ import annotations from pathlib import Path -from typing import TYPE_CHECKING, Type, cast +from typing import TYPE_CHECKING, cast from ipv8.bootstrapping.dispersy.bootstrapper import DispersyBootstrapper from ipv8.community import Community @@ -91,7 +91,7 @@ def get_overlay_class(self) -> type[Community]: """ Create a fake Community. """ - return cast(Type[Community], type(f"{self.__class__.__name__}", (Component,), {})) + return cast(type[Community], type(f"{self.__class__.__name__}", (Component,), {})) def get_my_peer(self, ipv8: IPv8, session: Session) -> Peer: """ @@ -138,16 +138,12 @@ def prepare(self, ipv8: IPv8, session: Session) -> None: Create the database instances we need for Tribler. """ from tribler.core.database.store import MetadataStore - from tribler.core.database.tribler_database import TriblerDatabase from tribler.core.notifier import Notification - db_path = str(Path(session.config.get_version_state_dir()) / "sqlite" / "tribler.db") mds_path = str(Path(session.config.get_version_state_dir()) / "sqlite" / "metadata.db") if session.config.get("memory_db"): - db_path = ":memory:" mds_path = ":memory:" - session.db = TriblerDatabase(db_path) session.mds = MetadataStore( mds_path, session.ipv8.keys["anonymous id"].key, @@ -166,7 +162,6 @@ def finalize(self, ipv8: IPv8, session: Session, community: Community) -> None: db_endpoint = session.rest_manager.get_endpoint("/api/metadata") db_endpoint.download_manager = session.download_manager db_endpoint.mds = session.mds - db_endpoint.tribler_db = session.db def get_endpoints(self) -> list[RESTEndpoint]: """ @@ -177,34 +172,6 @@ def get_endpoints(self) -> list[RESTEndpoint]: return [*super().get_endpoints(), DatabaseEndpoint()] -@set_in_session("knowledge_community") -@after("DatabaseComponent") -@precondition('session.config.get("database/enabled")') -@precondition('session.config.get("knowledge_community/enabled")') -@overlay("tribler.core.knowledge.community", "KnowledgeCommunity") -@kwargs(db="session.db", key='session.ipv8.keys["secondary"].key') -class KnowledgeComponent(BaseLauncher): - """ - Launch instructions for the knowledge community. - """ - - def finalize(self, ipv8: IPv8, session: Session, community: Community) -> None: - """ - When we are done launching, register our REST API. - """ - endpoint = session.rest_manager.get_endpoint("/api/knowledge") - endpoint.db = session.db - endpoint.community = community - - def get_endpoints(self) -> list[RESTEndpoint]: - """ - Add the knowledge endpoint. - """ - from tribler.core.knowledge.restapi.knowledge_endpoint import KnowledgeEndpoint - - return [*super().get_endpoints(), KnowledgeEndpoint()] - - @after("DatabaseComponent") @precondition('session.config.get("rendezvous/enabled")') @overlay("tribler.core.rendezvous.community", "RendezvousCommunity") diff --git a/src/tribler/core/content_discovery/community.py b/src/tribler/core/content_discovery/community.py index 733d6a7d89..dd39bebf48 100644 --- a/src/tribler/core/content_discovery/community.py +++ b/src/tribler/core/content_discovery/community.py @@ -8,7 +8,7 @@ from binascii import hexlify, unhexlify from importlib.metadata import PackageNotFoundError, version from itertools import count -from typing import TYPE_CHECKING, Any, Callable, Sequence +from typing import TYPE_CHECKING, Any, Callable from ipv8.community import Community, CommunitySettings from ipv8.lazy_community import lazy_wrapper @@ -24,18 +24,17 @@ VersionRequest, VersionResponse, ) -from tribler.core.database.layers.knowledge import ResourceType from tribler.core.database.orm_bindings.torrent_metadata import LZ4_EMPTY_ARCHIVE, entries_to_chunk from tribler.core.database.store import MetadataStore, ObjState, ProcessingResult -from tribler.core.knowledge.community import is_valid_resource from tribler.core.notifier import Notification, Notifier from tribler.core.torrent_checker.dataclasses import HealthInfo if TYPE_CHECKING: + from collections.abc import Sequence + from ipv8.types import Peer from tribler.core.database.orm_bindings.torrent_metadata import TorrentMetadata - from tribler.core.database.tribler_database import TriblerDatabase from tribler.core.torrent_checker.torrent_checker import TorrentChecker @@ -55,7 +54,6 @@ class ContentDiscoverySettings(CommunitySettings): metadata_store: MetadataStore torrent_checker: TorrentChecker - tribler_db: TriblerDatabase | None = None notifier: Notifier | None = None @@ -316,33 +314,8 @@ async def process_rpc_query(self, sanitized_parameters: dict[str, Any]) -> list: :raises ValueError: if no JSON could be decoded. :raises pony.orm.dbapiprovider.OperationalError: if an illegal query was performed. """ - if self.composition.tribler_db: - # tags should be extracted because `get_entries_threaded` doesn't expect them as a parameter - tags = sanitized_parameters.pop("tags", None) - - infohash_set = self.composition.tribler_db.instance(self.search_for_tags, tags) - if infohash_set: - sanitized_parameters["infohash_set"] = {bytes.fromhex(s) for s in infohash_set} - - # exclude_deleted should be extracted because `get_entries_threaded` doesn't expect it as a parameter - sanitized_parameters.pop("exclude_deleted", None) - return await self.composition.metadata_store.get_entries_threaded(**sanitized_parameters) - @db_session - def search_for_tags(self, tags: list[str] | None) -> set[str] | None: - """ - Query our local database for the given tags. - """ - if not tags or not self.composition.tribler_db: - return None - valid_tags = {tag for tag in tags if is_valid_resource(tag)} - return self.composition.tribler_db.knowledge.get_subjects_intersection( - subjects_type=ResourceType.TORRENT, - objects=valid_tags, - predicate=ResourceType.TAG, - case_sensitive=False - ) def send_db_results(self, peer: Peer, request_payload_id: int, db_results: list[TorrentMetadata]) -> None: """ diff --git a/src/tribler/core/database/layers/__init__.py b/src/tribler/core/database/layers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/tribler/core/database/layers/health.py b/src/tribler/core/database/layers/health.py deleted file mode 100644 index 721bf43f54..0000000000 --- a/src/tribler/core/database/layers/health.py +++ /dev/null @@ -1,171 +0,0 @@ -from __future__ import annotations - -import logging -from binascii import hexlify -from datetime import datetime -from enum import IntEnum -from typing import TYPE_CHECKING - -from pony import orm - -from tribler.core.database.layers.layer import EntityImpl, Layer - -if TYPE_CHECKING: - import dataclasses - - from pony.orm import Database - - from tribler.core.database.layers.knowledge import KnowledgeDataAccessLayer, Resource - from tribler.core.torrent_checker.dataclasses import HealthInfo - - @dataclasses.dataclass - class TorrentHealth(EntityImpl): - """ - Database type for torrent health information. - """ - - id: int - torrent: Resource - seeders: int - leechers: int - source: int - tracker: Tracker | None - last_check: datetime - - def __init__(self, torrent: Resource) -> None: ... # noqa: D107 - - @staticmethod - def get(torrent: Resource) -> TorrentHealth | None: ... # noqa: D102 - - @staticmethod - def get_for_update(torrent: Resource) -> TorrentHealth | None: ... # noqa: D102 - - @dataclasses.dataclass - class Tracker(EntityImpl): - """ - Database type for tracker definitions. - """ - - id: int - url: str - last_check: datetime | None - alive: bool - failures: int - torrents = set[Resource] - torrent_health_set = set[TorrentHealth] - - def __init__(self, url: str) -> None: ... # noqa: D107 - - @staticmethod - def get(url: str) -> Tracker | None: ... # noqa: D102 - - @staticmethod - def get_for_update(url: str) -> Tracker | None: ... # noqa: D102 - - -class ResourceType(IntEnum): - """ - Description of available resources within the Knowledge Graph. - These types are also using as a predicate for the statements. - - Based on https://en.wikipedia.org/wiki/Dublin_Core - """ - - CONTRIBUTOR = 1 - COVERAGE = 2 - CREATOR = 3 - DATE = 4 - DESCRIPTION = 5 - FORMAT = 6 - IDENTIFIER = 7 - LANGUAGE = 8 - PUBLISHER = 9 - RELATION = 10 - RIGHTS = 11 - SOURCE = 12 - SUBJECT = 13 - TITLE = 14 - TYPE = 15 - - # this is a section for extra types - TAG = 101 - TORRENT = 102 - CONTENT_ITEM = 103 - - -class HealthDataAccessLayer(Layer): - """ - A layer that stores health information. - """ - - def __init__(self, knowledge_layer: KnowledgeDataAccessLayer) -> None: - """ - Create a new health layer and initialize its bindings. - """ - self.logger = logging.getLogger(self.__class__.__name__) - self.instance = knowledge_layer.instance - self.Resource = knowledge_layer.Resource - self.TorrentHealth, self.Tracker = self.define_binding(self.instance) - - def get_torrent_health(self, infohash: str) -> TorrentHealth | None: - """ - Get the health belonging to the given infohash. - """ - if torrent := self.Resource.get(name=infohash, type=ResourceType.TORRENT): - return self.TorrentHealth.get(torrent=torrent) - return None - - @staticmethod - def define_binding(db: Database) -> tuple[type[TorrentHealth], type[Tracker]]: - """ - Create the bindings for this layer. - """ - class TorrentHealth(db.Entity): - id = orm.PrimaryKey(int, auto=True) - - torrent = orm.Required(lambda: db.Resource, index=True) - - seeders = orm.Required(int, default=0) - leechers = orm.Required(int, default=0) - source = orm.Required(int, default=0) # Source enum - tracker = orm.Optional(lambda: Tracker) - last_check = orm.Required(datetime, default=datetime.utcnow) - - class Tracker(db.Entity): - id = orm.PrimaryKey(int, auto=True) - - url = orm.Required(str, unique=True) - last_check = orm.Optional(datetime) - alive = orm.Required(bool, default=True) - failures = orm.Required(int, default=0) - - torrents = orm.Set(lambda: db.Resource) - torrent_health_set = orm.Set(lambda: TorrentHealth, reverse='tracker') - - return TorrentHealth, Tracker - - def add_torrent_health(self, health_info: HealthInfo) -> None: - """ - Store the given health info in the database. - """ - torrent = self.get_or_create( - self.Resource, - name=hexlify(health_info.infohash), - type=ResourceType.TORRENT - ) - - torrent_health = self.get_or_create( - self.TorrentHealth, - torrent=torrent - ) - - torrent_health.seeders = health_info.seeders - torrent_health.leechers = health_info.leechers - if health_info.tracker: - torrent_health.tracker = self.get_or_create( - self.Tracker, - url=health_info.tracker - ) - - torrent_health.source = health_info.source - torrent_health.last_check = datetime.utcfromtimestamp(health_info.last_check) # noqa: DTZ004 diff --git a/src/tribler/core/database/layers/knowledge.py b/src/tribler/core/database/layers/knowledge.py deleted file mode 100644 index c1644ab70f..0000000000 --- a/src/tribler/core/database/layers/knowledge.py +++ /dev/null @@ -1,559 +0,0 @@ -from __future__ import annotations - -import datetime -import logging -from dataclasses import dataclass -from enum import IntEnum -from typing import TYPE_CHECKING, Callable, Iterator, List, Set - -from pony import orm -from pony.orm import raw_sql -from pony.orm.core import Database, Entity, Query, UnrepeatableReadError, select -from pony.utils import between - -from tribler.core.database.layers.layer import EntityImpl, Layer -from tribler.core.knowledge.payload import StatementOperation - -CLOCK_START_VALUE = 0 - -PUBLIC_KEY_FOR_AUTO_GENERATED_OPERATIONS = b"auto_generated" - -SHOW_THRESHOLD = 1 # how many operation needed for showing a knowledge graph statement in the UI -HIDE_THRESHOLD = -2 # how many operation needed for hiding a knowledge graph statement in the UI - -if TYPE_CHECKING: - import dataclasses - - from tribler.core.database.layers.health import TorrentHealth, Tracker - - - @dataclasses.dataclass - class Peer(EntityImpl): - """ - Database type for a peer. - """ - - id: int - public_key: bytes - added_at: datetime.datetime | None - operations: set[StatementOp] - - def __init__(self, public_key: bytes) -> None: ... # noqa: D107 - - @staticmethod - def get(public_key: bytes) -> Peer | None: ... # noqa: D102 - - @staticmethod - def get_for_update(public_key: bytes) -> Peer | None: ... # noqa: D102 - - - @dataclasses.dataclass - class Statement(EntityImpl): - """ - Database type for a statement. - """ - - id: int - subject: Resource - object: Resource - operations: set[StatementOp] - added_count: int - removed_count: int - local_operation: int | None - - def __init__(self, subject: Resource, object: Resource) -> None: ... # noqa: D107, A002 - - @staticmethod - def get(subject: Resource, object: Resource) -> Statement | None: ... # noqa: D102, A002 - - @staticmethod - def get_for_update(subject: Resource, object: Resource) -> Statement | None: ... # noqa: D102, A002 - - - class IterResource(type): # noqa: D101 - - def __iter__(cls) -> Iterator[Resource]: ... # noqa: D105 - - - @dataclasses.dataclass - class Resource(EntityImpl, metaclass=IterResource): - """ - Database type for a resources. - """ - - id: int - name: str - type: int - subject_statements: set[Statement] - object_statements: set[Statement] - torrent_healths: set[TorrentHealth] - trackers: set[Tracker] - - def __init__(self, name: str, type: int) -> None: ... # noqa: D107, A002 - - @staticmethod - def get(name: str, type: int) -> Resource | None: ... # noqa: D102, A002 - - @staticmethod - def get_for_update(name: str, type: int) -> Resource | None: ... # noqa: D102, A002 - - - @dataclasses.dataclass - class StatementOp(EntityImpl): - """ - Database type for a statement operation. - """ - - id: int - statement: Statement - peer: Peer - operation: int - clock: int - signature: bytes - updated_at: datetime.datetime - auto_generated: bool - - def __init__(self, statement: Statement, peer: Peer, operation: int, clock: int, # noqa: D107 - signature: bytes, auto_generated: bool) -> None: ... - - @staticmethod - def get(statement: Statement, peer: Peer) -> StatementOp | None: ... # noqa: D102 - - @staticmethod - def get_for_update(statement: Statement, peer: Peer) -> StatementOp | None: ... # noqa: D102 - - -class Operation(IntEnum): - """ - Available types of statement operations. - """ - - ADD = 1 # +1 operation - REMOVE = 2 # -1 operation - - -class ResourceType(IntEnum): - """ - Description of available resources within the Knowledge Graph. - These types are also using as a predicate for the statements. - - Based on https://en.wikipedia.org/wiki/Dublin_Core - """ - - CONTRIBUTOR = 1 - COVERAGE = 2 - CREATOR = 3 - DATE = 4 - DESCRIPTION = 5 - FORMAT = 6 - IDENTIFIER = 7 - LANGUAGE = 8 - PUBLISHER = 9 - RELATION = 10 - RIGHTS = 11 - SOURCE = 12 - SUBJECT = 13 - TITLE = 14 - TYPE = 15 - - # this is a section for extra types - TAG = 101 - TORRENT = 102 - CONTENT_ITEM = 103 - - -@dataclass -class SimpleStatement: - """ - A statement that reflects some (typed) attribute ``object`` of a given (typed) ``subject``. - """ - - subject_type: int - subject: str - predicate: int - object: str - - -class KnowledgeDataAccessLayer(Layer): - """ - A database layer for knowledge. - """ - - def __init__(self, instance: orm.Database) -> None: - """ - Create a new knowledge database layer. - """ - self.logger = logging.getLogger(self.__class__.__name__) - self.instance = instance - self.Peer, self.Statement, self.Resource, self.StatementOp = self.define_binding(self.instance) - - @staticmethod - def define_binding(db: Database) -> tuple[type[Peer], type[Statement], type[Resource], type[StatementOp]]: - """ - Create the bindings for this layer. - """ - class Peer(db.Entity): - id = orm.PrimaryKey(int, auto=True) - public_key = orm.Required(bytes, unique=True) - added_at = orm.Optional(datetime.datetime, default=datetime.datetime.utcnow) - operations = orm.Set(lambda: StatementOp) - - class Statement(db.Entity): - id = orm.PrimaryKey(int, auto=True) - - subject = orm.Required(lambda: Resource) - object = orm.Required(lambda: Resource, index=True) - - operations = orm.Set(lambda: StatementOp) - - added_count = orm.Required(int, default=0) - removed_count = orm.Required(int, default=0) - - local_operation = orm.Optional(int) # in case user don't (or do) want to see it locally - - orm.composite_key(subject, object) - - @property - def score(self) -> int: - return self.added_count - self.removed_count - - def update_counter(self, operation: Operation, increment: int = 1, is_local_peer: bool = False) -> None: - """ - Update Statement's counter. - - :param operation: Resource operation. - :param increment: - :param is_local_peer: The flag indicates whether do we perform operations from a local user. - """ - if is_local_peer: - self.local_operation = operation - if operation == Operation.ADD: - self.added_count += increment - if operation == Operation.REMOVE: - self.removed_count += increment - - class Resource(db.Entity): - id = orm.PrimaryKey(int, auto=True) - name = orm.Required(str) - type = orm.Required(int) # ResourceType enum - - subject_statements = orm.Set(lambda: Statement, reverse="subject") - object_statements = orm.Set(lambda: Statement, reverse="object") - torrent_healths = orm.Set(lambda: db.TorrentHealth, reverse="torrent") - trackers = orm.Set(lambda: db.Tracker, reverse="torrents") - - orm.composite_key(name, type) - - class StatementOp(db.Entity): - id = orm.PrimaryKey(int, auto=True) - - statement = orm.Required(lambda: Statement) - peer = orm.Required(lambda: Peer) - - operation = orm.Required(int) - clock = orm.Required(int) - signature = orm.Required(bytes) - updated_at = orm.Required(datetime.datetime, default=datetime.datetime.utcnow) - auto_generated = orm.Required(bool, default=False) - - orm.composite_key(statement, peer) - - return Peer, Statement, Resource, StatementOp - - def _get_resources(self, resource_type: ResourceType | None, name: str | None, case_sensitive: bool) -> Query: - """ - Get resources. - - :param resource_type: type of resources - :param name: name of resources - :param case_sensitive: if True, then Resources are selected in a case-sensitive manner. - :returns: a Query object for requested resources - """ - results = self.Resource.select() - if name: - results = results.filter( - (lambda r: r.name == name) if case_sensitive else (lambda r: r.name.lower() == name.lower()) - ) - if resource_type: - results = results.filter(lambda r: r.type == resource_type.value) - return results - - def get_statements(self, source_type: ResourceType | None, source_name: str | None, - statements_getter: Callable[[Entity], Entity], - target_condition: Callable[[Statement], bool], condition: Callable[[Statement], bool], - case_sensitive: bool, ) -> Iterator[Statement]: - """ - Get entities that satisfies the given condition. - """ - for resource in self._get_resources(source_type, source_name, case_sensitive): - results = orm.select(_ for _ in statements_getter(resource) - .select(condition) - .filter(target_condition) - .order_by(lambda s: orm.desc(s.score))) - - yield from list(results) - - def add_operation(self, operation: StatementOperation, signature: bytes, is_local_peer: bool = False, - is_auto_generated: bool = False, counter_increment: int = 1) -> bool: - """ - Add the operation that will be applied to a statement. - - :param operation: the class describes the adding operation - :param signature: the signature of the operation - :param is_local_peer: local operations processes differently than remote operations. - :param is_auto_generated: the indicator of whether this resource was generated automatically or not - :param counter_increment: the counter or "numbers" of adding operations - :returns: True if the operation has been added/updated, False otherwise. - """ - self.logger.debug('Add operation. %s "%s" %s', - str(operation.subject), str(operation.predicate), str(operation.object)) - peer = self.get_or_create(self.Peer, public_key=operation.creator_public_key) - subject = self.get_or_create(self.Resource, name=operation.subject, type=operation.subject_type) - obj = self.get_or_create(self.Resource, name=operation.object, type=operation.predicate) - statement = self.get_or_create(self.Statement, subject=subject, object=obj) - op = self.StatementOp.get_for_update(statement=statement, peer=peer) - - if not op: # then insert - self.StatementOp(statement=statement, peer=peer, operation=operation.operation, - clock=operation.clock, signature=signature, auto_generated=is_auto_generated) - statement.update_counter(operation.operation, increment=counter_increment, is_local_peer=is_local_peer) - return True - - # if it is a message from the past, then return - if operation.clock <= op.clock: - return False - - # To prevent endless incrementing of the operation, we apply the following logic: - - # 1. Decrement previous operation - statement.update_counter(op.operation, increment=-counter_increment, is_local_peer=is_local_peer) - # 2. Increment new operation - statement.update_counter(operation.operation, increment=counter_increment, is_local_peer=is_local_peer) - - # 3. Update the operation entity - op.set(operation=operation.operation, clock=operation.clock, signature=signature, - updated_at=datetime.datetime.utcnow(), auto_generated=is_auto_generated) # noqa: DTZ003 - return True - - def add_auto_generated_operation(self, subject_type: ResourceType, subject: str, predicate: ResourceType, - obj: str) -> bool: - """ - Add an autogenerated operation. - - The difference between "normal" and "autogenerated" operation is that the autogenerated operation will be added - with the flag `is_auto_generated=True` and with the `PUBLIC_KEY_FOR_AUTO_GENERATED_TAGS` public key. - - :param subject_type: a type of adding subject. See: ResourceType enum. - :param subject: a string that represents a subject of adding operation. - :param predicate: the enum that represents a predicate of adding operation. - :param obj: a string that represents an object of adding operation. - """ - operation = StatementOperation( - subject_type=subject_type, - subject=subject, - predicate=predicate, - object=obj, - operation=Operation.ADD, - clock=CLOCK_START_VALUE, - creator_public_key=PUBLIC_KEY_FOR_AUTO_GENERATED_OPERATIONS, - ) - - return self.add_operation(operation, signature=b"", is_local_peer=False, is_auto_generated=True, - counter_increment=SHOW_THRESHOLD) - - @staticmethod - def _show_condition(s: Statement) -> bool: - """ - This function determines show condition for the statement. - """ - return s.local_operation == Operation.ADD.value or not s.local_operation and s.score >= SHOW_THRESHOLD - - def get_objects(self, subject_type: ResourceType | None = None, subject: str | None = "", - predicate: ResourceType | None = None, case_sensitive: bool = True, - condition: Callable[[Statement], bool] | None = None) -> List[str]: - """ - Get objects that satisfy the given subject and predicate. - - To understand the order of parameters, keep in ming the following generic construction: - (, , , ). - - So in the case of retrieving objects this construction becomes - (, , , ?). - - :param subject_type: a type of the subject. - :param subject: a string that represents the subject. - :param predicate: the enum that represents a predicate of querying operations. - :param case_sensitive: if True, then Resources are selected in a case-sensitive manner. - :returns: a list of the strings representing the objects. - """ - self.logger.debug("Get subjects for %s with %s", str(subject), str(predicate)) - - statements = self.get_statements( - source_type=subject_type, - source_name=subject, - statements_getter=lambda r: r.subject_statements, - target_condition=(lambda s: s.object.type == predicate.value) if predicate else (lambda _: True), - condition=condition or self._show_condition, - case_sensitive=case_sensitive, - ) - return [s.object.name for s in statements] - - def get_subjects(self, subject_type: ResourceType | None = None, predicate: ResourceType | None = None, - obj: str | None = "", case_sensitive: bool = True) -> List[str]: - """ - Get subjects that satisfy the given object and predicate. - - To understand the order of parameters, keep in mind the following generic construction: - - (, , , ). - - So in the case of retrieving subjects this construction becomes - (, ?, , ). - - :param subject_type: a type of the subject. - :param obj: a string that represents the object. - :param predicate: the enum that represents a predicate of querying operations. - :param case_sensitive: if True, then Resources are selected in a case-sensitive manner. - :returns: a list of the strings representing the subjects. - """ - self.logger.debug("Get linked back resources for %s with %s", str(obj), str(predicate)) - - statements = self.get_statements( - source_type=predicate, - source_name=obj, - statements_getter=lambda r: r.object_statements, - target_condition=(lambda s: s.subject.type == subject_type.value) if subject_type else (lambda _: True), - condition=self._show_condition, - case_sensitive=case_sensitive, - ) - - return [s.subject.name for s in statements] - - def get_simple_statements(self, subject_type: ResourceType | None = None, subject: str | None = "", - case_sensitive: bool = True) -> list[SimpleStatement]: - """ - Get simple statements for the given subject search. - """ - statements = self.get_statements( - source_type=subject_type, - source_name=subject, - statements_getter=lambda r: r.subject_statements, - target_condition=lambda _: True, - condition=self._show_condition, - case_sensitive=case_sensitive, - ) - - results = [] - for s in statements: - try: - results.append(SimpleStatement(subject_type=s.subject.type, subject=s.subject.name, - predicate=s.object.type, object=s.object.name)) - except UnrepeatableReadError as e: - self.logger.exception(e) - return results - - def get_suggestions(self, subject_type: ResourceType | None = None, subject: str | None = "", - predicate: ResourceType | None = None, case_sensitive: bool = True) -> List[str]: - """ - Get all suggestions for a particular subject. - - :param subject_type: a type of the subject. - :param subject: a string that represents the subject. - :param predicate: the enum that represents a predicate of querying operations. - :param case_sensitive: if True, then Resources are selected in a case-sensitive manner. - :returns: a list of the strings representing the objects. - """ - self.logger.debug("Getting suggestions for %s with %s", str(subject), str(predicate)) - - return self.get_objects( - subject_type=subject_type, - subject=subject, - predicate=predicate, - case_sensitive=case_sensitive, - condition=lambda s: not s.local_operation and between(s.score, HIDE_THRESHOLD + 1, SHOW_THRESHOLD - 1) - ) - - def get_subjects_intersection(self, objects: Set[str], - predicate: ResourceType | None, - subjects_type: ResourceType = ResourceType.TORRENT, - case_sensitive: bool = True) -> Set[str]: - """ - Get all subjects that have a certain predicate. - """ - if not objects: - return set() - - if case_sensitive: - name_condition = '"obj"."name" = $obj_name' - else: - name_condition = 'py_lower("obj"."name") = py_lower($obj_name)' - query = select(r.name for r in self.Resource if r.type == subjects_type.value) - for obj_name in objects: - query = query.filter(raw_sql(""" - r.id IN ( - SELECT "s"."subject" - FROM "Statement" "s" - WHERE ( - "s"."local_operation" = $(Operation.ADD.value) - OR - ("s"."local_operation" = 0 OR "s"."local_operation" IS NULL) - AND ("s"."added_count" - "s"."removed_count") >= $SHOW_THRESHOLD - ) AND "s"."object" IN ( - SELECT "obj"."id" FROM "Resource" "obj" - WHERE "obj"."type" = $(predicate.value) AND $name_condition - ) - )"""), globals={"obj_name": obj_name, "name_condition": name_condition, "SHOW_THRESHOLD": SHOW_THRESHOLD}) - return set(query) - - def get_clock(self, operation: StatementOperation) -> int: - """ - Get the clock (int) of operation. - """ - peer = self.Peer.get(public_key=operation.creator_public_key) - subject = self.Resource.get(name=operation.subject, type=operation.subject_type) - obj = self.Resource.get(name=operation.object, type=operation.predicate) - if not subject or not obj or not peer: - return CLOCK_START_VALUE - - statement = self.Statement.get(subject=subject, object=obj) - if not statement: - return CLOCK_START_VALUE - - op = self.StatementOp.get(statement=statement, peer=peer) - return op.clock if op else CLOCK_START_VALUE - - def get_operations_for_gossip(self, count: int = 10) -> set[Entity]: - """ - Get random operations from the DB. - - :param count: a limit for a resulting query - """ - return self._get_random_operations_by_condition( - condition=lambda so: not so.auto_generated, - count=count - ) - - def _get_random_operations_by_condition(self, condition: Callable[[Entity], bool], count: int = 5, - attempts: int = 100) -> set[Entity]: - """ - Get `count` random operations that satisfy the given condition. - - This method were introduce as an fast alternative for native Pony `random` method. - - :param condition: the condition by which the entities will be queried. - :param count: the amount of entities to return. - :param attempts: maximum attempt count for requesting the DB. - :returns: a set of random operations - """ - operations: set[Entity] = set() - for _ in range(attempts): - if len(operations) == count: - return operations - - random_operations_list = self.StatementOp.select_random(1) - if random_operations_list: - operation = random_operations_list[0] - if condition(operation): - operations.add(operation) - - return operations diff --git a/src/tribler/core/database/layers/layer.py b/src/tribler/core/database/layers/layer.py deleted file mode 100644 index 973a6f8f39..0000000000 --- a/src/tribler/core/database/layers/layer.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from typing import TypeVar - -from pony.orm.core import Entity - -EntityImpl = TypeVar("EntityImpl", bound=Entity) - - -class Layer: - """ - A generic database layer. - """ - - def get_or_create(self, cls: type[EntityImpl], create_kwargs: dict | None = None, **kwargs) -> EntityImpl: - """ - Get or create a db entity. - - :param cls: The Entity's class. - :param create_kwargs: Any necessary additional keyword arguments to create the entity. - :param kwargs: Keyword arguments to find the entity. - :returns: A new or existing instance. - """ - obj = cls.get_for_update(**kwargs) - if not obj: - if create_kwargs: - kwargs.update(create_kwargs) - obj = cls(**kwargs) - return obj diff --git a/src/tribler/core/database/restapi/database_endpoint.py b/src/tribler/core/database/restapi/database_endpoint.py index 2971759c23..45705bfccf 100644 --- a/src/tribler/core/database/restapi/database_endpoint.py +++ b/src/tribler/core/database/restapi/database_endpoint.py @@ -4,7 +4,6 @@ import json import typing from binascii import unhexlify -from dataclasses import asdict from aiohttp import web from aiohttp_apispec import docs, querystring_schema @@ -13,14 +12,12 @@ from pony.orm import db_session from typing_extensions import Self, TypeAlias -from tribler.core.database.layers.knowledge import ResourceType from tribler.core.database.queries import to_fts_query from tribler.core.database.restapi.schema import MetadataSchema, SearchMetadataParameters, TorrentSchema from tribler.core.database.serialization import REGULAR_TORRENT from tribler.core.notifier import Notification from tribler.core.restapi.rest_endpoint import ( HTTP_BAD_REQUEST, - HTTP_NOT_FOUND, MAX_REQUEST_SIZE, RESTEndpoint, RESTResponse, @@ -30,7 +27,6 @@ from multidict import MultiDictProxy, MultiMapping from tribler.core.database.store import MetadataStore - from tribler.core.database.tribler_database import TriblerDatabase from tribler.core.libtorrent.download_manager.download_manager import DownloadManager from tribler.core.restapi.rest_manager import TriblerRequest from tribler.core.torrent_checker.torrent_checker import TorrentChecker @@ -87,7 +83,6 @@ def __init__(self, middlewares: tuple = (), client_max_size: int = MAX_REQUEST_S self.download_manager: DownloadManager | None = None self.torrent_checker: TorrentChecker | None = None - self.tribler_db: TriblerDatabase | None = None self.app.add_routes( [ @@ -126,23 +121,6 @@ def sanitize_parameters(cls: type[Self], sanitized["sort_by"] = "HEALTH" return sanitized - @db_session - def add_statements_to_metadata_list(self, contents_list: list[dict]) -> None: - """ - Load statements from the database and attach them to the torrent descriptions in the content list. - """ - if self.tribler_db is None: - self._logger.error("Cannot add statements to metadata list: tribler_db is not set in %s", - self.__class__.__name__) - return - for torrent in contents_list: - if torrent["type"] == REGULAR_TORRENT: - raw_statements = self.tribler_db.knowledge.get_simple_statements( - subject_type=ResourceType.TORRENT, - subject=torrent["infohash"] - ) - torrent["statements"] = [asdict(stmt) for stmt in raw_statements] - @docs( tags=["Metadata"], summary="Fetch the swarm health of a specific torrent.", @@ -237,7 +215,6 @@ async def get_popular_torrents(self, request: RequestType) -> RESTResponse: contents_list = [entry.to_simple_dict() for entry in request.context[0].get_entries(**sanitized)] self.add_download_progress_to_metadata_list(contents_list) - self.add_statements_to_metadata_list(contents_list) response_dict = { "results": contents_list, "first": sanitized["first"], @@ -265,25 +242,18 @@ async def get_popular_torrents(self, request: RequestType) -> RESTResponse: }, ) @querystring_schema(SearchMetadataParameters) - async def local_search(self, request: RequestType) -> RESTResponse: # noqa: C901 + async def local_search(self, request: RequestType) -> RESTResponse: """ Perform a search for a given query. """ try: sanitized = self.sanitize_parameters(request.query) - tags = sanitized.pop("tags", None) except (ValueError, KeyError): return RESTResponse({"error": { "handled": True, "message": "Error processing request parameters" }}, status=HTTP_BAD_REQUEST) - if self.tribler_db is None: - return RESTResponse({"error": { - "handled": True, - "message": "Tribler DB not initialized" - }}, status=HTTP_NOT_FOUND) - include_total = request.query.get("include_total", "") query = request.query.get("fts_text") if query is None: @@ -315,23 +285,11 @@ def search_db() -> tuple[list[dict], int, int]: return search_results, total, max_rowid try: - with db_session: - if tags: - infohash_set = self.tribler_db.knowledge.get_subjects_intersection( - subjects_type=ResourceType.TORRENT, - objects=set(typing.cast(list[str], tags)), - predicate=ResourceType.TAG, - case_sensitive=False) - if infohash_set: - sanitized["infohash_set"] = {bytes.fromhex(s) for s in infohash_set} - search_results, total, max_rowid = await mds.run_threaded(search_db) except Exception as e: self._logger.exception("Error while performing DB search: %s: %s", type(e).__name__, e) return RESTResponse(status=HTTP_BAD_REQUEST) - self.add_statements_to_metadata_list(search_results) - response_dict = { "results": search_results, "first": sanitized["first"], diff --git a/src/tribler/core/database/tribler_database.py b/src/tribler/core/database/tribler_database.py deleted file mode 100644 index 71a9cee76a..0000000000 --- a/src/tribler/core/database/tribler_database.py +++ /dev/null @@ -1,135 +0,0 @@ -from __future__ import annotations - -import logging -import os -from pathlib import Path -from typing import TYPE_CHECKING, cast - -from pony import orm -from pony.orm import Database, db_session - -from tribler.core.database.layers.health import HealthDataAccessLayer -from tribler.core.database.layers.knowledge import KnowledgeDataAccessLayer - -if TYPE_CHECKING: - import dataclasses - - - @dataclasses.dataclass - class Misc: - """ - A miscellaneous key value mapping in the database. - """ - - name: str - value: str | None - - def __init__(self, name: str) -> None: ... # noqa: D107 - - @staticmethod - def get(name: str) -> Misc | None: ... # noqa: D102 - - @staticmethod - def get_for_update(name: str) -> Misc | None: ... # noqa: D102 - -MEMORY = ":memory:" - - -class TriblerDatabase: - """ - A wrapper for the Tribler database. - """ - - CURRENT_VERSION = 1 - _SCHEME_VERSION_KEY = "scheme_version" - - def __init__(self, filename: str | None = None, *, create_tables: bool = True, **generate_mapping_kwargs) -> None: - """ - Create a new tribler database. - """ - self.instance = Database() - - self.knowledge = KnowledgeDataAccessLayer(self.instance) - self.health = HealthDataAccessLayer(self.knowledge) - - self.Misc = self.define_binding(self.instance) - - self.Peer = self.knowledge.Peer - self.Statement = self.knowledge.Statement - self.Resource = self.knowledge.Resource - self.StatementOp = self.knowledge.StatementOp - - self.TorrentHealth = self.health.TorrentHealth - self.Tracker = self.health.Tracker - - filename = filename or MEMORY - db_does_not_exist = filename == MEMORY or not os.path.isfile(filename) - - if filename != MEMORY: - Path(filename).parent.mkdir(parents=True, exist_ok=True) - - self.instance.bind(provider='sqlite', filename=filename, create_db=db_does_not_exist) - generate_mapping_kwargs['create_tables'] = create_tables - self.instance.generate_mapping(**generate_mapping_kwargs) - self.logger = logging.getLogger(self.__class__.__name__) - - if db_does_not_exist: - self.fill_default_data() - - @staticmethod - def define_binding(db: Database) -> type[Misc]: - """ - Define common bindings. - """ - - class Misc(db.Entity): - name = orm.PrimaryKey(str) - value = orm.Optional(str) - - return Misc - - @db_session - def fill_default_data(self) -> None: - """ - Add a misc entry for the database version. - """ - self.logger.info("Filling the DB with the default data") - self.set_misc(self._SCHEME_VERSION_KEY, str(self.CURRENT_VERSION)) - - def get_misc(self, key: str, default: str | None = None) -> str | None: - """ - Retrieve a value from the database or return the default value if it is not found. - """ - data = self.Misc.get(name=key) - return data.value if data else default - - def set_misc(self, key: str, value: str) -> None: - """ - Set or add the value of a given key. - """ - obj = self.Misc.get_for_update(name=key) or self.Misc(name=key) - obj.value = value - - @property - def version(self) -> int: - """ - Get the database version. - """ - return int(cast(str, self.get_misc(key=self._SCHEME_VERSION_KEY, default="0"))) - - @version.setter - def version(self, value: int) -> None: - """ - Set the database version. - """ - if not isinstance(value, int): - msg = "DB version should be integer" - raise TypeError(msg) - - self.set_misc(key=self._SCHEME_VERSION_KEY, value=str(value)) - - def shutdown(self) -> None: - """ - Disconnect from the database. - """ - self.instance.disconnect() diff --git a/src/tribler/core/knowledge/__init__.py b/src/tribler/core/knowledge/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/tribler/core/knowledge/community.py b/src/tribler/core/knowledge/community.py deleted file mode 100644 index a6d611b1a5..0000000000 --- a/src/tribler/core/knowledge/community.py +++ /dev/null @@ -1,242 +0,0 @@ -from __future__ import annotations - -import random -from binascii import unhexlify -from typing import TYPE_CHECKING - -from cryptography.exceptions import InvalidSignature -from ipv8.community import Community, CommunitySettings -from ipv8.lazy_community import lazy_wrapper -from pony.orm import db_session - -from tribler.core.database.layers.knowledge import Operation, ResourceType -from tribler.core.knowledge.operations_requests import OperationsRequests, PeerValidationError -from tribler.core.knowledge.payload import ( - RawStatementOperationMessage, - RequestStatementOperationMessage, - StatementOperation, - StatementOperationMessage, - StatementOperationSignature, -) - -if TYPE_CHECKING: - from ipv8.keyvault.private.libnaclkey import LibNaCLSK - from ipv8.types import Key, Peer - - from tribler.core.database.tribler_database import TriblerDatabase - -REQUESTED_OPERATIONS_COUNT = 10 -CLEAR_ALL_REQUESTS_INTERVAL = 10 * 60 # 10 minutes - - -class KnowledgeCommunitySettings(CommunitySettings): - """ - Settings for the knowledge community. - """ - - db: TriblerDatabase - key: LibNaCLSK - request_interval: int = 5 - - -class KnowledgeCommunity(Community): - """ - Community for disseminating knowledge across the network. - """ - - community_id = unhexlify("d7f7bdc8bcd3d9ad23f06f25aa8aab6754eb23a0") - settings_class = KnowledgeCommunitySettings - - def __init__(self, settings: KnowledgeCommunitySettings) -> None: - """ - Create a new knowledge community. - """ - super().__init__(settings) - self.db = settings.db - self.key = settings.key - self.requests = OperationsRequests() - - self.cool_peers: list[Peer] | None = None - - self.add_message_handler(RawStatementOperationMessage, self.on_message) - self.add_message_handler(RequestStatementOperationMessage, self.on_request) - - self.register_task("request_operations", self.request_operations, interval=settings.request_interval) - self.register_task("clear_requests", self.requests.clear_requests, interval=CLEAR_ALL_REQUESTS_INTERVAL) - self.logger.info("Knowledge community initialized") - - def get_cool_peers(self) -> list[Peer]: - """ - We may need to freeze the peer list in this community to avoid inflating the peer count. - - Peers sampled from the frozen list are "cool" peers. - """ - known_peers = self.get_peers() - if self.max_peers < 0 or len(known_peers) < self.max_peers + 5: - self.cool_peers = None - return known_peers - # We may not be frozen yet and old cool peers may have gone offline. - if self.cool_peers is None or len(self.cool_peers) <= len(known_peers) // 2: - cool_peers = known_peers[:self.max_peers] - else: - cool_peers = self.cool_peers - self.cool_peers = [p for p in cool_peers if p in known_peers] - return self.cool_peers - - def request_operations(self) -> None: - """ - Contact peers to request operations. - """ - if not self.get_peers(): - return - - peer = random.choice(self.get_cool_peers()) - self.requests.register_peer(peer, REQUESTED_OPERATIONS_COUNT) - self.logger.info("-> request %d operations from peer %s", REQUESTED_OPERATIONS_COUNT, peer.mid.hex()) - self.ez_send(peer, RequestStatementOperationMessage(count=REQUESTED_OPERATIONS_COUNT)) - - @lazy_wrapper(RawStatementOperationMessage) - def on_message(self, peer: Peer, raw: RawStatementOperationMessage) -> None: - """ - Callback for when a raw statement operation message is received. - """ - if peer not in self.get_cool_peers(): - self.logger.debug("Dropping message from %s: peer is not cool!", str(peer)) - return - - operation, _ = self.serializer.unpack_serializable(StatementOperation, raw.operation) - signature, _ = self.serializer.unpack_serializable(StatementOperationSignature, raw.signature) - self.logger.debug("<- message received: %s", str(operation)) - try: - remote_key = self.crypto.key_from_public_bin(operation.creator_public_key) - - self.requests.validate_peer(peer) - self.verify_signature(packed_message=raw.operation, key=remote_key, signature=signature.signature, - operation=operation) - self.validate_operation(operation) - - with db_session(serializable=True): - is_added = self.db.knowledge.add_operation(operation, signature.signature) - if is_added: - s = f"+ operation added ({operation.object!r} \"{operation.predicate}\" {operation.subject!r})" - self.logger.info(s) - - except PeerValidationError as e: # peer has exhausted his response count - self.logger.warning(e) - except ValueError as e: # validation error - self.logger.warning(e) - except InvalidSignature as e: # signature verification error - self.logger.warning(e) - - @lazy_wrapper(RequestStatementOperationMessage) - def on_request(self, peer: Peer, operation: RequestStatementOperationMessage) -> None: - """ - Callback for when statement operations are requested. - """ - if peer not in self.get_cool_peers(): - self.logger.debug("Dropping message from %s: peer is not cool!", str(peer)) - return - - operations_count = min(max(1, operation.count), REQUESTED_OPERATIONS_COUNT) - self.logger.debug("<- peer %s requested %d operations", peer.mid.hex(), operations_count) - - with db_session: - random_operations = self.db.knowledge.get_operations_for_gossip(count=operations_count) - - self.logger.debug("Response %d operations", len(random_operations)) - sent_operations = [] - for op in random_operations: - try: - operation = StatementOperation( - subject_type=op.statement.subject.type, - subject=op.statement.subject.name, - predicate=op.statement.object.type, - object=op.statement.object.name, - operation=op.operation, - clock=op.clock, - creator_public_key=op.peer.public_key, - ) - self.validate_operation(operation) - signature = StatementOperationSignature(signature=op.signature) - self.ez_send(peer, StatementOperationMessage(operation=operation, signature=signature)) - sent_operations.append(operation) - except ValueError as e: # validation error - self.logger.warning(e) - if sent_operations: - sent_tags_info = ", ".join(f"({t})" for t in sent_operations) - self.logger.debug("-> sent operations (%s) to peer: %s", sent_tags_info, peer.mid.hex()) - - @staticmethod - def validate_operation(operation: StatementOperation) -> None: - """ - Check if an operation is valid and raise an exception if it is not. - - :raises ValueError: If the operation failed to validate. - """ - validate_resource(operation.subject) - validate_resource(operation.object) - validate_operation(operation.operation) - validate_resource_type(operation.subject_type) - validate_resource_type(operation.predicate) - - def verify_signature(self, packed_message: bytes, key: Key, signature: bytes, - operation: StatementOperation) -> None: - """ - Check if a signature is valid for the given message and raise an exception if it is not. - - :raises InvalidSignature: If the message is not correctly signed. - """ - if not self.crypto.is_valid_signature(key, packed_message, signature): - msg = f"Invalid signature for {operation}" - raise InvalidSignature(msg) - - def sign(self, operation: StatementOperation) -> bytes: - """ - Sign the given operation using our key. - """ - packed = self.serializer.pack_serializable(operation) - return self.crypto.create_signature(self.key, packed) - - -def validate_resource(resource: str) -> None: - """ - Validate the resource. - - :raises ValueError: If the case the resource is not valid. - """ - if len(resource) < MIN_RESOURCE_LENGTH or len(resource) > MAX_RESOURCE_LENGTH: - msg = f"Tag length should be in range [{MIN_RESOURCE_LENGTH}..{MAX_RESOURCE_LENGTH}]" - raise ValueError(msg) - - -def is_valid_resource(resource: str) -> bool: - """ - Validate the resource. Returns False, in the case the resource is not valid. - """ - try: - validate_resource(resource) - except ValueError: - return False - return True - - -def validate_operation(operation: int) -> None: - """ - Validate the incoming operation. - - :raises ValueError: If the case the operation is not valid. - """ - Operation(operation) - - -def validate_resource_type(t: int) -> None: - """ - Validate the resource type. - - :raises ValueError: If the case the type is not valid. - """ - ResourceType(t) - - -MIN_RESOURCE_LENGTH = 2 -MAX_RESOURCE_LENGTH = 50 diff --git a/src/tribler/core/knowledge/content_bundling.py b/src/tribler/core/knowledge/content_bundling.py deleted file mode 100644 index d06dff4f14..0000000000 --- a/src/tribler/core/knowledge/content_bundling.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -import logging -import math -import re -from collections import defaultdict -from itertools import chain -from typing import Dict, Iterable, TypedDict, cast - -logger = logging.getLogger(__name__) - - -class DictWithName(TypedDict): - """ - A dictionary that has a "name" key. - """ - - name: str - - -def _words_pattern(min_word_length: int = 3) -> str: - return r"[^\W\d_]{" + str(min_word_length) + ",}" - - -def _create_name(content_list: list[DictWithName], number: str, min_word_length: int = 4) -> str: - """ - Create a name for a group of content items based on the most common word in the title. - If several most frequently occurring words are found, preference is given to the longest word. - - :param content_list: list of content items - :param number: group number - :param min_word_length: minimum word length to be considered as a candidate for the group name - :returns: created group name. The name is capitalized. - """ - words: defaultdict[str, int] = defaultdict(int) - for item in content_list: - pattern = _words_pattern(min_word_length) - title_words = {w.lower() for w in re.findall(pattern, item["name"]) if w} - for word in title_words: - words[word] += 1 - if not words: - return number - m = max(words.values()) - candidates = (k for k, v in words.items() if v == m) - longest_word = max(candidates, key=len) - name = f"{longest_word} {number}" - return name[0].capitalize() + name[1:] - - -def calculate_diversity(content_list: Iterable[DictWithName], min_word_length: int = 4) -> float: - """ - Calculate the diversity of words in the titles of the content list. - The diversity calculation based on Corrected Type-Token Ratio (CTTR) formula. - - :param content_list: list of content items. Each item should have a "name" key with a title. - :param min_word_length: minimum word length to be considered as a word in the title. - :returns: diversity of words in the titles - """ - pattern = _words_pattern(min_word_length) - titles = (item["name"] for item in content_list) - words_in_titles = (re.findall(pattern, title) for title in titles) - words = [w.lower() for w in chain.from_iterable(words_in_titles) if w] - total_words = len(words) - if total_words == 0: - return 0 - unique_words = set(words) - - return len(unique_words) / math.sqrt(2 * total_words) - - -def group_content_by_number(content_list: Iterable[dict], - min_group_size: int = 2) -> Dict[str, list[DictWithName]]: - """ - Group content by the first number in the title. Returned groups keep the order in which it was found in the input. - - :param content_list: list of content items. Each item should have a "name" key with a title. - :param min_group_size: minimum number of content items in a group. In the case of a group with fewer items, it will - not be included in the result. - :returns: group number as key and list of content items as value - """ - groups: defaultdict[str, list[DictWithName]] = defaultdict(list) - for item in content_list: - if "name" in item and (m := re.search(r"\d+", item["name"])): - first_number = m.group(0).lstrip("0") or "0" - groups[first_number].append(cast(DictWithName, item)) - - filtered_groups = ((k, v) for k, v in groups.items() if len(v) >= min_group_size) - return {_create_name(v, k): v for k, v in filtered_groups} diff --git a/src/tribler/core/knowledge/operations_requests.py b/src/tribler/core/knowledge/operations_requests.py deleted file mode 100644 index 3c1c2ea4e7..0000000000 --- a/src/tribler/core/knowledge/operations_requests.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -from collections import defaultdict -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from ipv8.types import Peer - - -class PeerValidationError(ValueError): - """ - A peer has exceeded their number of responses. - """ - - -class OperationsRequests: - """ - This class is design for controlling requests during pull-based gossip. - - The main idea: - * Before a request, a client registered a peer with some number of expected responses - * While a response, the controller decrements number of expected responses for this peer - * The controller validates response by checking that expected responses for this peer is greater then 0 - """ - - def __init__(self) -> None: - """ - Create a new dictionary to keep track of responses. - """ - self.requests: dict[Peer, int] = defaultdict(int) - - def register_peer(self, peer: Peer, number_of_responses: int) -> None: - """ - Set the number of allowed responses for a given peer. - """ - self.requests[peer] = number_of_responses - - def validate_peer(self, peer: Peer) -> None: - """ - Decrement the number of responses of a Peer and check if the given peer has exceeded their allowed responses. - - :raises PeerValidationError: When the given peer has less than 0 responses remaining. - """ - if self.requests[peer] <= 0: - msg = f"Peer has exhausted his response count {peer}" - raise PeerValidationError(msg) - - self.requests[peer] -= 1 - - def clear_requests(self) -> None: - """ - Reset all allowed responses for all peers to 0. - """ - self.requests = defaultdict(int) diff --git a/src/tribler/core/knowledge/payload.py b/src/tribler/core/knowledge/payload.py deleted file mode 100644 index 5ea1fe85e8..0000000000 --- a/src/tribler/core/knowledge/payload.py +++ /dev/null @@ -1,74 +0,0 @@ -from ipv8.messaging.lazy_payload import VariablePayload, vp_compile - - -@vp_compile -class StatementOperation(VariablePayload): - """ - Do not change the format of the StatementOperation, because this will result in an invalid signature. - """ - - names = ["subject_type", "subject", "predicate", "object", "operation", "clock", "creator_public_key"] - format_list = ["q", "varlenHutf8", "q", "varlenHutf8", "q", "q", "74s"] - - subject_type: int # ResourceType enum - subject: str - predicate: int # ResourceType enum - object: str - operation: int # Operation enum - clock: int # This is the lamport-like clock that unique for each quadruple {public_key, subject, predicate, object} - creator_public_key: bytes - - -@vp_compile -class StatementOperationSignature(VariablePayload): - """ - A single signature of an unknown statement. - """ - - names = ["signature"] - format_list = ["64s"] - - signature: bytes - - -@vp_compile -class RawStatementOperationMessage(VariablePayload): - """ - RAW payload class is used for reducing ipv8 unpacking operations. - - For more information take a look at: https://github.com/Tribler/tribler/pull/6396#discussion_r728334323 - """ - - names = ["operation", "signature"] - format_list = ["varlenH", "varlenH"] - - operation: bytes - signature: bytes - msg_id = 2 - - -@vp_compile -class StatementOperationMessage(VariablePayload): - """ - An operation coupled to a signature. - """ - - names = ["operation", "signature"] - format_list = [StatementOperation, StatementOperationSignature] - - operation: bytes - signature: bytes - msg_id = 2 - - -@vp_compile -class RequestStatementOperationMessage(VariablePayload): - """ - Request a given number of statements. - """ - - names = ["count"] - format_list = ["q"] - - count: int - msg_id = 1 diff --git a/src/tribler/core/knowledge/restapi/__init__.py b/src/tribler/core/knowledge/restapi/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/tribler/core/knowledge/restapi/knowledge_endpoint.py b/src/tribler/core/knowledge/restapi/knowledge_endpoint.py deleted file mode 100644 index 2433d7c658..0000000000 --- a/src/tribler/core/knowledge/restapi/knowledge_endpoint.py +++ /dev/null @@ -1,175 +0,0 @@ -from __future__ import annotations - -import binascii -from typing import TYPE_CHECKING - -from aiohttp import web -from aiohttp_apispec import docs -from ipv8.REST.schema import schema -from marshmallow import Schema -from marshmallow.fields import Boolean, List, String -from pony.orm import db_session -from typing_extensions import TypeAlias - -from tribler.core.database.layers.knowledge import Operation, ResourceType -from tribler.core.knowledge.community import KnowledgeCommunity, is_valid_resource -from tribler.core.knowledge.payload import StatementOperation -from tribler.core.restapi.rest_endpoint import HTTP_BAD_REQUEST, MAX_REQUEST_SIZE, RESTEndpoint, RESTResponse - -if TYPE_CHECKING: - from tribler.core.database.tribler_database import TriblerDatabase - from tribler.core.restapi.rest_manager import TriblerRequest - - RequestType: TypeAlias = TriblerRequest[tuple[TriblerDatabase, KnowledgeCommunity]] - - -class HandledErrorSchema(Schema): - """ - The REST schema for knowledge errors. - """ - - error = String(description="Optional field describing any failures that may have occurred", required=True) - - -class KnowledgeEndpoint(RESTEndpoint): - """ - Top-level endpoint for knowledge management. - """ - - path = "/api/knowledge" - - def __init__(self, middlewares: tuple = (), client_max_size: int = MAX_REQUEST_SIZE) -> None: - """ - Create a new knowledge endpoint. - """ - super().__init__(middlewares, client_max_size) - - self.db: TriblerDatabase | None = None - self.required_components = ("db", ) - - self.community: KnowledgeCommunity | None = None - - self.app.add_routes( - [ - web.patch("/{infohash}", self.update_knowledge_entries), - web.get("/{infohash}/tag_suggestions", self.get_tag_suggestions), - ] - ) - - @staticmethod - def validate_infohash(infohash: str) -> tuple[bool, RESTResponse | None]: - """ - Check if the given bytes are a string of 40 HEX-character bytes. - """ - try: - if len(infohash) != 40: - return False, RESTResponse({"error": { - "handled": True, - "message": "Invalid infohash" - }}, status=HTTP_BAD_REQUEST) - except binascii.Error: - return False, RESTResponse({"error": { - "handled": True, - "message": "Invalid infohash" - }}, status=HTTP_BAD_REQUEST) - - return True, None - - @docs( - tags=["General"], - summary="Update the metadata associated with a particular torrent.", - responses={ - 200: { - "schema": schema(UpdateTagsResponse={"success": Boolean()}) - }, - HTTP_BAD_REQUEST: { - "schema": HandledErrorSchema, "example": {"error": {"handled": True, "message": "Invalid tag length"}} - } - }, - description="This endpoint updates a particular torrent with the provided metadata." - ) - async def update_knowledge_entries(self, request: RequestType) -> RESTResponse: - """ - Update the metadata associated with a particular torrent. - """ - params = await request.json() - infohash = request.match_info["infohash"] - ih_valid, error_response = KnowledgeEndpoint.validate_infohash(infohash) - if not ih_valid: - return error_response - - # Validate whether the size of the tag is within the allowed range and filter out duplicate tags. - statements = [] - self._logger.info("Statements about %s: %s", infohash, str(params["statements"])) - for statement in params["statements"]: - obj = statement["object"] - if not is_valid_resource(obj): - return RESTResponse({"error": { - "handled": True, - "message": "Invalid tag length" - }}, status=HTTP_BAD_REQUEST) - - statements.append(statement) - - self.modify_statements(request.context[0], infohash, statements) - - return RESTResponse({"success": True}) - - @db_session - def modify_statements(self, db: TriblerDatabase, infohash: str, statements: list) -> None: - """ - Modify the statements of a particular content item. - """ - if not self.community: - return - - # First, get the current statements and compute the diff between the old and new statements - old_statements = db.knowledge.get_statements(subject_type=ResourceType.TORRENT, subject=infohash) - old_statements = {(stmt.predicate, stmt.object) for stmt in old_statements} - self._logger.info("Old statements: %s", old_statements) - new_statements = {(stmt["predicate"], stmt["object"]) for stmt in statements} - self._logger.info("New statements: %s", new_statements) - added_statements = new_statements - old_statements - removed_statements = old_statements - new_statements - - # Create individual statement operations for the added/removed statements - public_key = self.community.key.pub().key_to_bin() - for stmt in added_statements.union(removed_statements): - predicate, obj = stmt - type_of_operation = Operation.ADD if stmt in added_statements else Operation.REMOVE - operation = StatementOperation(subject_type=ResourceType.TORRENT, subject=infohash, - predicate=predicate, - object=obj, operation=type_of_operation, clock=0, - creator_public_key=public_key) - operation.clock = db.knowledge.get_clock(operation) + 1 - signature = self.community.sign(operation) - db.knowledge.add_operation(operation, signature, is_local_peer=True) - - self._logger.info("Added statements: %s", added_statements) - self._logger.info("Removed statements: %s", removed_statements) - - @docs( - tags=["General"], - summary="Get tag suggestions for a torrent with a particular infohash.", - responses={ - 200: { - "schema": schema(SuggestedTagsResponse={"suggestions": List(String)}) - }, - HTTP_BAD_REQUEST: { - "schema": HandledErrorSchema, "example": {"error": {"handled": True, "message": "Invalid infohash"}} - } - }, - description="This endpoint updates a particular torrent with the provided tags." - ) - async def get_tag_suggestions(self, request: RequestType) -> RESTResponse: - """ - Get suggested tags for a particular torrent. - """ - infohash = request.match_info["infohash"] - ih_valid, error_response = KnowledgeEndpoint.validate_infohash(infohash) - if not ih_valid: - return error_response - - with db_session: - suggestions = request.context[0].knowledge.get_suggestions(subject=infohash, predicate=ResourceType.TAG) - return RESTResponse({"suggestions": suggestions}) diff --git a/src/tribler/core/libtorrent/restapi/create_torrent_endpoint.py b/src/tribler/core/libtorrent/restapi/create_torrent_endpoint.py index eb47a0672c..eb3ecd107f 100644 --- a/src/tribler/core/libtorrent/restapi/create_torrent_endpoint.py +++ b/src/tribler/core/libtorrent/restapi/create_torrent_endpoint.py @@ -11,7 +11,6 @@ from ipv8.REST.schema import schema from marshmallow.fields import String -from tribler.core.knowledge.restapi.knowledge_endpoint import HandledErrorSchema from tribler.core.libtorrent.download_manager.download_config import DownloadConfig from tribler.core.libtorrent.download_manager.download_manager import DownloadManager from tribler.core.libtorrent.torrentdef import TorrentDef @@ -74,7 +73,7 @@ def __init__(self, download_manager: DownloadManager, client_max_size: int = MAX "examples": {"Success": {"success": True}} }, HTTP_BAD_REQUEST: { - "schema": HandledErrorSchema, + "schema": schema(HandledErrorSchema={"error": "any failures that may have occurred"}), "examples": {"Error": {"error": {"handled": True, "message": "files parameter missing"}}} } } diff --git a/src/tribler/core/session.py b/src/tribler/core/session.py index f492743fc4..4546e62ce6 100644 --- a/src/tribler/core/session.py +++ b/src/tribler/core/session.py @@ -18,7 +18,6 @@ ContentDiscoveryComponent, DatabaseComponent, DHTDiscoveryComponent, - KnowledgeComponent, RecommenderComponent, RendezvousComponent, TorrentCheckerComponent, @@ -48,7 +47,6 @@ from types import TracebackType from tribler.core.database.store import MetadataStore - from tribler.core.database.tribler_database import TriblerDatabase from tribler.core.torrent_checker.torrent_checker import TorrentChecker from tribler.tribler_config import TriblerConfigManager @@ -148,7 +146,6 @@ def __init__(self, config: TriblerConfigManager) -> None: self.rest_manager = RESTManager(self.config) # Optional globals, set by components: - self.db: TriblerDatabase | None = None self.mds: MetadataStore | None = None self.torrent_checker: TorrentChecker | None = None @@ -156,7 +153,7 @@ def register_launchers(self) -> None: """ Register all IPv8 launchers that allow communities to be loaded. """ - for launcher_class in [ContentDiscoveryComponent, DatabaseComponent, DHTDiscoveryComponent, KnowledgeComponent, + for launcher_class in [ContentDiscoveryComponent, DatabaseComponent, DHTDiscoveryComponent, RecommenderComponent, RendezvousComponent, TorrentCheckerComponent, TunnelComponent, VersioningComponent, WatchFolderComponent]: instance = launcher_class() @@ -281,9 +278,6 @@ async def shutdown(self) -> None: await server.stop() # Stop database activities - if self.db: - self.notifier.notify(Notification.tribler_shutdown_state, state="Shutting down general-purpose database.") - self.db.shutdown() if self.mds: self.notifier.notify(Notification.tribler_shutdown_state, state="Shutting down metadata database.") self.mds.shutdown() diff --git a/src/tribler/test_unit/core/content_discovery/test_community.py b/src/tribler/test_unit/core/content_discovery/test_community.py index f3bd013375..0a29bc1345 100644 --- a/src/tribler/test_unit/core/content_discovery/test_community.py +++ b/src/tribler/test_unit/core/content_discovery/test_community.py @@ -20,7 +20,6 @@ VersionRequest, VersionResponse, ) -from tribler.core.database.layers.knowledge import ResourceType from tribler.core.database.orm_bindings.torrent_metadata import LZ4_EMPTY_ARCHIVE from tribler.core.database.serialization import REGULAR_TORRENT from tribler.core.notifier import Notification, Notifier @@ -221,36 +220,6 @@ async def test_request_for_version_build(self) -> None: self.assertEqual(sys.platform, message.platform) self.assertEqual("Tribler 1.2.3", message.version) - def test_search_for_tags_no_db(self) -> None: - """ - Test if search_for_tags returns None without tribler_db. - """ - self.overlay(0).composition.tribler_db = None - - self.assertIsNone(self.overlay(0).search_for_tags(tags=['tag'])) - - def test_search_for_tags_no_tags(self) -> None: - """ - Test if search_for_tags returns None without tags. - """ - self.overlay(0).composition.tribler_db = None - - self.assertIsNone(self.overlay(0).search_for_tags(tags=[])) - - def test_search_for_tags_only_valid_tags(self) -> None: - """ - Test if search_for_tags filters valid tags. - """ - args = {} - self.overlay(0).composition.tribler_db = Mock(knowledge=Mock(get_subjects_intersection=args.update)) - - self.overlay(0).search_for_tags(tags=['invalid_tag' * 50, 'valid_tag']) - - self.assertEqual(ResourceType.TORRENT, args["subjects_type"]) - self.assertEqual({'valid_tag'}, args["objects"]) - self.assertEqual(ResourceType.TAG, args["predicate"]) - self.assertEqual(False, args["case_sensitive"]) - async def test_process_rpc_query(self) -> None: """ Test if process_rpc_query searches the TriblerDB and MetadataStore. @@ -262,7 +231,6 @@ async def test_process_rpc_query(self) -> None: await self.overlay(0).process_rpc_query({'first': 0, 'infohash_set': None, 'last': 100}) self.assertEqual(0, async_mock.call_args.kwargs["first"]) - self.assertEqual({b"\x01" * 20}, async_mock.call_args.kwargs["infohash_set"]) self.assertEqual(100, async_mock.call_args.kwargs["last"]) async def test_remote_select(self) -> None: diff --git a/src/tribler/test_unit/core/database/layers/__init__.py b/src/tribler/test_unit/core/database/layers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/tribler/test_unit/core/database/layers/test_health.py b/src/tribler/test_unit/core/database/layers/test_health.py deleted file mode 100644 index 9a357338be..0000000000 --- a/src/tribler/test_unit/core/database/layers/test_health.py +++ /dev/null @@ -1,111 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace -from typing import TYPE_CHECKING - -from ipv8.test.base import TestBase - -from tribler.core.database.layers.health import HealthDataAccessLayer, ResourceType -from tribler.core.torrent_checker.dataclasses import HealthInfo, Source - -if TYPE_CHECKING: - from typing_extensions import Self - - -class MockResource: - """ - A mocked Resource that stored is call kwargs. - """ - - def __init__(self, **kwargs) -> None: - """ - Create a MockResouce and store its kwargs. - """ - self.get_kwargs = kwargs - - @classmethod - def get(cls: type[Self], **kwargs) -> type[Self]: - """ - Fake a search using the given kwargs and return an instance of ourselves. - """ - return cls(**kwargs) - - @classmethod - def get_for_update(cls: type[Self], /, **kwargs) -> type[Self] | None: - """ - Mimic fetching the resource from the database. - """ - del kwargs - return None - - - -class MockEntity(MockResource, SimpleNamespace): - """ - Allow a db binding to write whatever they want to this class. - """ - - CREATED = [] - - def __init__(self, **kwargs) -> None: - """ - Create a new MockEntity and add it to the CREATED list. - """ - super().__init__(**kwargs) - self.CREATED.append(self) - - -class MockDatabase: - """ - Mock the bindings that others will inherit from. - """ - - Entity = MockEntity - Resource = MockResource - - -class MockKnowledgeDataAccessLayer: - """ - A mocked KnowledgeDataAccessLayer. - """ - - def __init__(self) -> None: - """ - Create a new mocked KnowledgeDataAccessLayer. - """ - self.instance = MockDatabase() - self.Resource = self.instance.Resource - self.instance.Entity.CREATED = [] - - -class TestHealthDataAccessLayer(TestBase): - """ - Tests for the HealthDataAccessLayer. - """ - - def test_get_torrent_health(self) -> None: - """ - Test if torrents with the correct infohash are retrieved for their health info. - """ - hdal = HealthDataAccessLayer(MockKnowledgeDataAccessLayer()) - - health = hdal.get_torrent_health("01" * 20) - - self.assertEqual("01" * 20, health.get_kwargs["torrent"].get_kwargs["name"]) - self.assertEqual(ResourceType.TORRENT, health.get_kwargs["torrent"].get_kwargs["type"]) - - def test_add_torrent_health(self) -> None: - """ - Test if adding torrent health leads to the correct database calls. - """ - hdal = HealthDataAccessLayer(MockKnowledgeDataAccessLayer()) - hdal.TorrentHealth = MockEntity - - hdal.add_torrent_health(HealthInfo(b"\x01" * 20, 7, 42, 1337)) - added, = hdal.TorrentHealth.CREATED - - self.assertEqual(b"01" * 20, added.get_kwargs["torrent"].get_kwargs["name"]) - self.assertEqual(ResourceType.TORRENT, added.get_kwargs["torrent"].get_kwargs["type"]) - self.assertEqual(7, added.seeders) - self.assertEqual(42, added.leechers) - self.assertEqual(Source.UNKNOWN, added.source) diff --git a/src/tribler/test_unit/core/database/layers/test_knowledge.py b/src/tribler/test_unit/core/database/layers/test_knowledge.py deleted file mode 100644 index 3005476c75..0000000000 --- a/src/tribler/test_unit/core/database/layers/test_knowledge.py +++ /dev/null @@ -1,289 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace -from typing import TYPE_CHECKING - -from ipv8.test.base import TestBase - -from tribler.core.database.layers.health import ResourceType -from tribler.core.database.layers.knowledge import KnowledgeDataAccessLayer, Operation -from tribler.core.knowledge.payload import StatementOperation - -if TYPE_CHECKING: - from typing_extensions import Self - - -class MockResource: - """ - A mocked Resource that stored is call kwargs. - """ - - def __init__(self, **kwargs) -> None: - """ - Create a MockResouce and store its kwargs. - """ - self.get_kwargs = kwargs - - @classmethod - def get(cls: type[Self], **kwargs) -> type[Self]: - """ - Fake a search using the given kwargs and return an instance of ourselves. - """ - return cls(**kwargs) - - @classmethod - def get_for_update(cls: type[Self], /, **kwargs) -> type[Self] | None: - """ - Mimic fetching the resource from the database. - """ - return cls(**kwargs) - - -class MockEntity(MockResource, SimpleNamespace): - """ - Allow a db binding to write whatever they want to this class. - """ - - CREATED = [] - - def __init__(self, **kwargs) -> None: - """ - Create a new MockEntity and add it to the CREATED list. - """ - super().__init__(**kwargs) - self.CREATED.append(self) - - -class MockStatement(MockEntity): - """ - A mocked Statement. - """ - - def update_counter(self, operation: Operation, increment: int = 1, is_local_peer: bool = False) -> None: - """ - Fake a counter update and store the calling args. - """ - self.update_counter_arg_op = operation - self.update_counter_arg_increment = increment - self.update_counter_arg_local_peer = is_local_peer - - -class MockStatementOp(MockEntity): - """ - A mocked StatementOp. - """ - - clock = 0 - operation = Operation.ADD - - def set(self, **kwargs) -> None: - """ - Fake a set and store the calling args. - """ - self.set_kwargs = kwargs - - def __hash__(self) -> int: - """ - We need a hash. - """ - return 0 - - @classmethod - def select_random(cls: type[Self], count: int) -> list[Self]: - """ - Fake random selection. - """ - out = [] - for i in range(count): - statement_op = MockStatementOp() - statement_op.id = i - statement_op.statement = None - statement_op.peer = None - statement_op.operation = Operation.ADD.value - statement_op.clock = 0 - statement_op.signature = b"" - statement_op.updated_at = 0 - statement_op.auto_generated = False - out.append(statement_op) - return out - - -class MockStatementOpMissing(MockStatement): - """ - A mocked StatementOp that does not exist. - """ - - @classmethod - def get_for_update(cls: type[Self], /, **kwargs) -> type[Self] | None: - """ - It did not exist before. - """ - del kwargs - return None - - -class MockDatabase: - """ - Mock the bindings that others will inherit from. - """ - - Entity = MockEntity - Resource = MockResource - - -class TestKnowledgeDataAccessLayer(TestBase): - """ - Tests for the KnowledgeDataAccessLayer. - """ - - def setUp(self) -> None: - """ - Mock all bindings. - """ - super().setUp() - self.kdal = KnowledgeDataAccessLayer(MockDatabase()) - self.kdal.Statement = MockStatement - self.kdal.Statement.CREATED = [] - self.kdal.StatementOp = MockStatementOpMissing - self.kdal.StatementOp.CREATED = [] - - def get_created(self, entity_type: type[MockEntity]) -> list[MockEntity]: - """ - Get all instances of a particular mock entity type that have been created. - """ - return [entity for entity in entity_type.CREATED if isinstance(entity, entity_type)] - - def add_searchable_statement(self, subj: str = "\x01" * 20, obj: str = "test tag") -> None: - """ - Inject a Statement that can be searched for. - """ - self.kdal.add_auto_generated_operation(ResourceType.TORRENT, subj, ResourceType.TAG, obj) - statement, = self.get_created(self.kdal.Statement) - statement.subject_statements = self.kdal.Statement - statement.object = SimpleNamespace() - statement.object.name = statement.get_kwargs["object"].get_kwargs["name"] - statement.object.type = statement.get_kwargs["object"].get_kwargs["type"] - statement.subject = SimpleNamespace() - statement.subject.name = statement.get_kwargs["subject"].get_kwargs["name"] - statement.subject.type = statement.get_kwargs["subject"].get_kwargs["type"] - self.kdal.get_statements = lambda **kwargs: [statement] - - def test_add_operation_update(self) -> None: - """ - Test if operations are correctly updated in the ORM. - """ - statement_operation = StatementOperation(subject_type=ResourceType.TORRENT, subject=b"\x01" * 20, - predicate=ResourceType.TAG, object='test tag', - operation=Operation.ADD, clock=1, creator_public_key=b"\x02" * 64) - self.kdal.StatementOp = MockStatementOp - self.kdal.StatementOp.CREATED = [] - value = self.kdal.add_operation(statement_operation, b"\x00" * 32, True) - - statement, = self.get_created(self.kdal.Statement) - statement_op, = self.get_created(self.kdal.StatementOp) - - self.assertTrue(value) - self.assertEqual(b"\x01" * 20, statement.get_kwargs["subject"].get_kwargs["name"]) - self.assertEqual(ResourceType.TORRENT, statement.get_kwargs["subject"].get_kwargs["type"]) - self.assertEqual("test tag", statement.get_kwargs["object"].get_kwargs["name"]) - self.assertEqual(ResourceType.TAG, statement.get_kwargs["object"].get_kwargs["type"]) - self.assertEqual(Operation.ADD, statement.update_counter_arg_op) - self.assertEqual(1, statement.update_counter_arg_increment) - self.assertTrue(statement.update_counter_arg_local_peer) - self.assertEqual(Operation.ADD, statement_op.set_kwargs["operation"]) - self.assertEqual(1, statement_op.set_kwargs["clock"]) - self.assertEqual(b"\x00" * 32, statement_op.set_kwargs["signature"]) - self.assertFalse(statement_op.set_kwargs["auto_generated"]) - - def test_add_operation_past(self) -> None: - """ - Test if operations are not added to the ORM if their clock is in the past. - """ - statement_operation = StatementOperation(subject_type=ResourceType.TORRENT, subject=b"\x01" * 20, - predicate=ResourceType.TAG, object='test tag', - operation=Operation.ADD, clock=-1, creator_public_key=b"\x02" * 64) - self.kdal.StatementOp = MockStatementOp - self.kdal.StatementOp.CREATED = [] - value = self.kdal.add_operation(statement_operation, b"\x00" * 32, True) - - self.assertFalse(value) - - def test_add_operation_missing(self) -> None: - """ - Test if operations are correctly added to the ORM is the statement op did not exist before. - """ - statement_operation = StatementOperation(subject_type=ResourceType.TORRENT, subject=b"\x01" * 20, - predicate=ResourceType.TAG, object='test tag', - operation=Operation.ADD, clock=1, creator_public_key=b"\x02" * 64) - value = self.kdal.add_operation(statement_operation, b"\x00" * 32, True) - - statement, = self.get_created(self.kdal.Statement) - - self.assertTrue(value) - self.assertEqual(b"\x01" * 20, statement.get_kwargs["subject"].get_kwargs["name"]) - self.assertEqual(ResourceType.TORRENT, statement.get_kwargs["subject"].get_kwargs["type"]) - self.assertEqual("test tag", statement.get_kwargs["object"].get_kwargs["name"]) - self.assertEqual(ResourceType.TAG, statement.get_kwargs["object"].get_kwargs["type"]) - self.assertEqual(Operation.ADD, statement.update_counter_arg_op) - self.assertEqual(1, statement.update_counter_arg_increment) - self.assertTrue(statement.update_counter_arg_local_peer) - - def test_add_auto_generated_operation(self) -> None: - """ - Test if auto generated operations are correctly added to the ORM. - """ - value = self.kdal.add_auto_generated_operation(ResourceType.TORRENT, "\x01" * 20, ResourceType.TAG, 'test tag') - - statement, = self.get_created(self.kdal.Statement) - - self.assertTrue(value) - self.assertEqual("\x01" * 20, statement.get_kwargs["subject"].get_kwargs["name"]) - self.assertEqual(ResourceType.TORRENT, statement.get_kwargs["subject"].get_kwargs["type"]) - self.assertEqual("test tag", statement.get_kwargs["object"].get_kwargs["name"]) - self.assertEqual(ResourceType.TAG, statement.get_kwargs["object"].get_kwargs["type"]) - self.assertEqual(Operation.ADD, statement.update_counter_arg_op) - self.assertEqual(1, statement.update_counter_arg_increment) - self.assertFalse(statement.update_counter_arg_local_peer) - - def test_get_objects(self) -> None: - """ - Test if objects are correctly retrieved from the ORM. - """ - self.add_searchable_statement(obj="test tag") - - value = self.kdal.get_objects() - - self.assertEqual(["test tag"], value) - - def test_get_subjects(self) -> None: - """ - Test if subjects are correctly retrieved from the ORM. - """ - self.add_searchable_statement(subj="\x01" * 20) - - value = self.kdal.get_subjects() - - self.assertEqual(["\x01" * 20], value) - - def test_get_suggestions(self) -> None: - """ - Test if suggestions are correctly retrieved from the ORM. - """ - self.add_searchable_statement(subj="\x01" * 20, obj="test tag") - - value = self.kdal.get_suggestions(subject="\x01" * 20) - - self.assertEqual(["test tag"], value) - - def test_get_operations_for_gossip(self) -> None: - """ - Test if. - """ - self.kdal.StatementOp = MockStatementOp - - selected, = self.kdal.get_operations_for_gossip(1) - - self.assertFalse(selected.auto_generated) - self.assertEqual(0, selected.clock) - self.assertEqual(0, selected.id) - self.assertEqual(Operation.ADD.value, selected.operation) diff --git a/src/tribler/test_unit/core/database/layers/test_layer.py b/src/tribler/test_unit/core/database/layers/test_layer.py deleted file mode 100644 index f96c7ecac8..0000000000 --- a/src/tribler/test_unit/core/database/layers/test_layer.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from ipv8.test.base import TestBase - -from tribler.core.database.layers.layer import Layer - -if TYPE_CHECKING: - from typing_extensions import Self - - -class MockEntity: - """ - A mocked-up database entity. - """ - - def __init__(self, /, **kwargs) -> None: - """ - Create a new MockEntity and store the kwargs. - """ - self.init_kwargs = kwargs - - @classmethod - def get_for_update(cls: type[Self], /, **kwargs) -> type[Self] | None: - """ - Mimic fetching the entity from the database. - """ - return cls(**kwargs) - - -class MockUnknownEntity(MockEntity): - """ - A mocked-up database entity that cannot be retrieved. - """ - - @classmethod - def get_for_update(cls: type[Self], /, **kwargs) -> type[Self] | None: - """ - Mimic fetching the entity from the database. - """ - del kwargs - return None - - -class TestLayer(TestBase): - """ - Tests for the Layer base class. - """ - - def test_retrieve_existing(self) -> None: - """ - Test retrieving a known entity. - """ - layer = Layer() - - value = layer.get_or_create(MockEntity) - - self.assertIsNotNone(value) - self.assertEqual(value.init_kwargs, {}) - - def test_create_no_kwargs_no_create(self) -> None: - """ - Test creating a new entity without kwargs or create kwargs. - """ - layer = Layer() - - value = layer.get_or_create(MockUnknownEntity) - - self.assertIsNotNone(value) - self.assertEqual(value.init_kwargs, {}) - - def test_create_no_kwargs_with_create(self) -> None: - """ - Test creating a new entity without kwargs but with create kwargs. - """ - layer = Layer() - - value = layer.get_or_create(MockUnknownEntity, create_kwargs={"a": 1}) - - self.assertIsNotNone(value) - self.assertEqual(value.init_kwargs, {"a": 1}) - - def test_create_with_kwargs_no_create(self) -> None: - """ - Test creating a new entity with kwargs but without create kwargs. - """ - layer = Layer() - - value = layer.get_or_create(MockUnknownEntity, a=1) - - self.assertIsNotNone(value) - self.assertEqual(value.init_kwargs, {"a": 1}) - - def test_create_with_kwargs_with_create(self) -> None: - """ - Test creating a new entity with both kwargs and create kwargs. - """ - layer = Layer() - - value = layer.get_or_create(MockUnknownEntity, create_kwargs={"a": 1}, b=2) - - self.assertIsNotNone(value) - self.assertEqual(value.init_kwargs, {"a": 1, "b": 2}) diff --git a/src/tribler/test_unit/core/database/restapi/test_database_endpoint.py b/src/tribler/test_unit/core/database/restapi/test_database_endpoint.py index 2a76d380fd..6524f759e1 100644 --- a/src/tribler/test_unit/core/database/restapi/test_database_endpoint.py +++ b/src/tribler/test_unit/core/database/restapi/test_database_endpoint.py @@ -8,7 +8,6 @@ from ipv8.test.REST.rest_base import MockRequest, response_to_json from multidict import MultiDict, MultiDictProxy -from tribler.core.database.layers.knowledge import ResourceType, SimpleStatement from tribler.core.database.restapi.database_endpoint import DatabaseEndpoint, parse_bool from tribler.core.database.serialization import REGULAR_TORRENT from tribler.core.restapi.rest_endpoint import HTTP_BAD_REQUEST @@ -57,22 +56,6 @@ def test_parse_bool(self) -> None: self.assertFalse(parse_bool("false")) self.assertFalse(parse_bool("0")) - def test_add_statements_to_metadata_list(self) -> None: - """ - Test if statements can be added to an existing metadata dict. - """ - metadata = {"type": REGULAR_TORRENT, "infohash": "AA"} - endpoint = DatabaseEndpoint() - endpoint.tribler_db = Mock(knowledge=Mock(get_simple_statements=Mock(return_value=[ - SimpleStatement(ResourceType.TORRENT, "AA", ResourceType.TAG, "tag") - ]))) - endpoint.add_statements_to_metadata_list([metadata]) - - self.assertEqual(ResourceType.TORRENT, metadata["statements"][0]["subject_type"]) - self.assertEqual("AA", metadata["statements"][0]["subject"]) - self.assertEqual(ResourceType.TAG, metadata["statements"][0]["predicate"]) - self.assertEqual("tag", metadata["statements"][0]["object"]) - async def test_get_torrent_health_bad_timeout(self) -> None: """ Test if a bad timeout value in get_torrent_health leads to a HTTP_BAD_REQUEST status. @@ -153,39 +136,6 @@ def test_add_download_progress_to_metadata_list_metainfo_requests(self) -> None: self.assertNotIn("progress", metadata) - async def test_get_popular_torrents(self) -> None: - """ - Test if we can bring everything together into a popular torrents request. - - Essentially, this combines ``add_download_progress_to_metadata_list`` and ``add_statements_to_metadata_list``. - """ - metadata = {"type": REGULAR_TORRENT, "infohash": "AA"} - endpoint = DatabaseEndpoint() - endpoint.tribler_db = Mock(knowledge=Mock(get_simple_statements=Mock(return_value=[ - SimpleStatement(ResourceType.TORRENT, "AA", ResourceType.TAG, "tag") - ]))) - download = Mock(get_state=Mock(return_value=Mock(get_progress=Mock(return_value=1.0))), - tdef=Mock(infohash="AA")) - endpoint.download_manager = Mock(get_download=Mock(return_value=download), metainfo_requests=[]) - endpoint.mds = Mock(get_entries=Mock(return_value=[Mock(to_simple_dict=Mock(return_value=metadata))])) - request = MockRequest("/api/metadata/torrents/popular") - request.context = [endpoint.mds] - - response = await endpoint.get_popular_torrents(request) - response_body_json = await response_to_json(response) - response_results = response_body_json["results"][0] - - self.assertEqual(200, response.status) - self.assertEqual(1, response_body_json["first"]) - self.assertEqual(50, response_body_json["last"]) - self.assertEqual(300, response_results["type"]) - self.assertEqual("AA", response_results["infohash"]) - self.assertEqual(1.0, response_results["progress"]) - self.assertEqual(ResourceType.TORRENT.value, response_results["statements"][0]["subject_type"]) - self.assertEqual("AA", response_results["statements"][0]["subject"]) - self.assertEqual(ResourceType.TAG.value, response_results["statements"][0]["predicate"]) - self.assertEqual("tag", response_results["statements"][0]["object"]) - async def test_local_search_bad_query(self) -> None: """ Test if a bad value leads to a bad request status. @@ -213,9 +163,9 @@ async def test_local_search_errored_search(self) -> None: self.assertEqual(HTTP_BAD_REQUEST, response.status) - async def test_local_search_no_knowledge(self) -> None: + async def test_local_search(self) -> None: """ - Test if performing a local search without a tribler db set returns mds results. + Test if performing a local search returns mds results. """ endpoint = DatabaseEndpoint() endpoint.tribler_db = Mock() @@ -235,7 +185,7 @@ async def test_local_search_no_knowledge(self) -> None: self.assertEqual(None, response_body_json["sort_by"]) self.assertEqual(True, response_body_json["sort_desc"]) - async def test_local_search_no_knowledge_include_total(self) -> None: + async def test_local_search_include_total(self) -> None: """ Test if performing a local search with requested total, includes a total. """ diff --git a/src/tribler/test_unit/core/database/test_tribler_database.py b/src/tribler/test_unit/core/database/test_tribler_database.py deleted file mode 100644 index 14fdbbec4e..0000000000 --- a/src/tribler/test_unit/core/database/test_tribler_database.py +++ /dev/null @@ -1,60 +0,0 @@ -from ipv8.test.base import TestBase -from pony.orm import db_session - -from tribler.core.database.tribler_database import TriblerDatabase - - -class TestTriblerDatabase(TestBase): - """ - Tests for the TriblerDatabase class. - """ - - def setUp(self) -> None: - """ - Create a database instance. - """ - super().setUp() - self.db = TriblerDatabase(":memory:") - - @db_session - def test_set_misc(self) -> None: - """ - Test if set_misc works as expected. - """ - self.db.set_misc(key='string', value='value') - self.db.set_misc(key='integer', value="1") - - self.assertEqual("value", self.db.get_misc(key='string')) - self.assertEqual("1", self.db.get_misc(key='integer')) - - @db_session - def test_non_existent_misc(self) -> None: - """ - Test if get_misc returns proper values. - """ - self.assertIsNone(self.db.get_misc(key="non existent")) - self.assertEqual("42", self.db.get_misc(key="non existent", default="42")) - - @db_session - def test_default_version(self) -> None: - """ - Test if the default version is equal to ``CURRENT_VERSION``. - """ - self.assertEqual(TriblerDatabase.CURRENT_VERSION, self.db.version) - - @db_session - def test_version_getter_and_setter(self) -> None: - """ - Test if the version getter and setter work as expected. - """ - self.db.version = 42 - - self.assertEqual(42, self.db.version) - - @db_session - def test_version_getter_unsupported_type(self) -> None: - """ - Test if the version getter raises a TypeError if the type is not supported. - """ - with self.assertRaises(TypeError): - self.db.version = 'string' diff --git a/src/tribler/test_unit/core/knowledge/__init__.py b/src/tribler/test_unit/core/knowledge/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/tribler/test_unit/core/knowledge/restapi/__init__.py b/src/tribler/test_unit/core/knowledge/restapi/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/tribler/test_unit/core/knowledge/restapi/test_knowledge_endpoint.py b/src/tribler/test_unit/core/knowledge/restapi/test_knowledge_endpoint.py deleted file mode 100644 index 12f85fb115..0000000000 --- a/src/tribler/test_unit/core/knowledge/restapi/test_knowledge_endpoint.py +++ /dev/null @@ -1,165 +0,0 @@ -from unittest.mock import Mock - -from ipv8.keyvault.crypto import default_eccrypto -from ipv8.peer import Peer -from ipv8.peerdiscovery.network import Network -from ipv8.test.base import TestBase -from ipv8.test.mocking.endpoint import AutoMockEndpoint -from ipv8.test.REST.rest_base import MockRequest, response_to_json - -from tribler.core.database.layers.knowledge import ResourceType -from tribler.core.knowledge.community import KnowledgeCommunity, KnowledgeCommunitySettings -from tribler.core.knowledge.payload import StatementOperation -from tribler.core.knowledge.restapi.knowledge_endpoint import KnowledgeEndpoint - - -class MockCommunity(KnowledgeCommunity): - """ - An inert KnowledgeCommunity. - """ - - community_id = b"\x00" * 20 - - def __init__(self, settings: KnowledgeCommunitySettings) -> None: - """ - Create a new MockCommunity. - """ - super().__init__(settings) - self.cancel_all_pending_tasks() - - def sign(self, operation: StatementOperation) -> bytes: - """ - Fake a signature. - """ - return b"" - - -class TestKnowledgeEndpoint(TestBase): - """ - Tests for the KnowledgeEndpoint REST endpoint. - """ - - def setUp(self) -> None: - """ - Create a new endpoint and a mock community. - """ - super().setUp() - key = default_eccrypto.generate_key("curve25519") - settings = KnowledgeCommunitySettings( - endpoint=AutoMockEndpoint(), - my_peer=Peer(key), - network=Network(), - key=key, - db=Mock() - ) - self.endpoint = KnowledgeEndpoint() - self.endpoint.db = settings.db - self.endpoint.community = MockCommunity(settings) - - def tag_to_statement(self, tag: str) -> dict: - """ - Convert a tag to a statement dictionary. - """ - return {"predicate": ResourceType.TAG, "object": tag} - - async def test_add_tag_invalid_infohash(self) -> None: - """ - Test if an error is returned if we try to add a tag to content with an invalid infohash. - """ - post_data = {"knowledge": [self.tag_to_statement("abc"), self.tag_to_statement("def")]} - request = MockRequest("/api/knowledge/3f3", "PATCH", post_data, {"infohash": "3f3"}) - request.context = [self.endpoint.db] - - response = await self.endpoint.update_knowledge_entries(request) - response_body_json = await response_to_json(response) - - self.assertEqual(400, response.status) - self.assertEqual("Invalid infohash", response_body_json["error"]["message"]) - - async def test_add_invalid_tag_too_short(self) -> None: - """ - Test whether an error is returned if we try to add a tag that is too short or long. - """ - post_data = {"statements": [self.tag_to_statement("a")]} - request = MockRequest("/api/knowledge/" + "a" * 40, "PATCH", post_data, {"infohash": "a" * 40}) - request.context = [self.endpoint.db] - - response = await self.endpoint.update_knowledge_entries(request) - response_body_json = await response_to_json(response) - - self.assertEqual(400, response.status) - self.assertEqual("Invalid tag length", response_body_json["error"]["message"]) - - async def test_add_invalid_tag_too_long(self) -> None: - """ - Test whether an error is returned if we try to add a tag that is too short or long. - """ - post_data = {"statements": [self.tag_to_statement("a" * 60)]} - request = MockRequest("/api/knowledge/" + "a" * 40, "PATCH", post_data, {"infohash": "a" * 40}) - request.context = [self.endpoint.db] - - response = await self.endpoint.update_knowledge_entries(request) - response_body_json = await response_to_json(response) - - self.assertEqual(400, response.status) - self.assertEqual("Invalid tag length", response_body_json["error"]["message"]) - - async def test_modify_tags(self) -> None: - """ - Test modifying tags. - """ - post_data = {"statements": [self.tag_to_statement("abc"), self.tag_to_statement("def")]} - self.endpoint.db.knowledge.get_statements = Mock(return_value=[]) - self.endpoint.db.knowledge.get_clock = Mock(return_value=0) - request = MockRequest("/api/knowledge/" + "a" * 40, "PATCH", post_data, {"infohash": "a" * 40}) - request.context = [self.endpoint.db] - - response = await self.endpoint.update_knowledge_entries(request) - response_body_json = await response_to_json(response) - - self.assertEqual(200, response.status) - self.assertTrue(response_body_json["success"]) - - async def test_modify_tags_no_community(self) -> None: - """ - Test if the KnowledgeEndpoint can function without a community. - """ - self.endpoint.community = None - post_data = {"statements": [self.tag_to_statement("abc"), self.tag_to_statement("def")]} - self.endpoint.db.knowledge.get_statements = Mock(return_value=[]) - self.endpoint.db.knowledge.get_clock = Mock(return_value=0) - request = MockRequest("/api/knowledge/" + "a" * 40, "PATCH", post_data, {"infohash": "a" * 40}) - request.context = [self.endpoint.db] - - response = await self.endpoint.update_knowledge_entries(request) - response_body_json = await response_to_json(response) - - self.assertEqual(200, response.status) - self.assertTrue(response_body_json["success"]) - - async def test_get_suggestions_invalid_infohash(self) -> None: - """ - Test if an error is returned if we fetch suggestions from content with an invalid infohash. - """ - request = MockRequest("/api/knowledge/3f3/tag_suggestions", match_info={"infohash": "3f3"}) - request.context = [self.endpoint.db] - - response = await self.endpoint.get_tag_suggestions(request) - response_body_json = await response_to_json(response) - - self.assertEqual(400, response.status) - self.assertEqual("Invalid infohash", response_body_json["error"]["message"]) - - async def test_get_suggestions(self) -> None: - """ - Test if we can successfully fetch suggestions from content. - """ - self.endpoint.db.knowledge.get_suggestions = Mock(return_value=["test"]) - request = MockRequest("/api/knowledge/" + "a" * 40 + "/tag_suggestions", match_info={"infohash": "a" * 40}) - request.context = [self.endpoint.db] - - response = await self.endpoint.get_tag_suggestions(request) - response_body_json = await response_to_json(response) - - self.assertEqual(200, response.status) - self.assertEqual(["test"], response_body_json["suggestions"]) diff --git a/src/tribler/test_unit/core/knowledge/test_community.py b/src/tribler/test_unit/core/knowledge/test_community.py deleted file mode 100644 index a519271eed..0000000000 --- a/src/tribler/test_unit/core/knowledge/test_community.py +++ /dev/null @@ -1,257 +0,0 @@ -from __future__ import annotations - -import dataclasses -from datetime import datetime, timezone -from random import sample -from typing import TYPE_CHECKING -from unittest.mock import Mock - -from ipv8.keyvault.crypto import default_eccrypto -from ipv8.test.base import TestBase -from pony.orm import db_session - -from tribler.core.database.layers.knowledge import Operation, ResourceType -from tribler.core.knowledge.community import ( - KnowledgeCommunity, - KnowledgeCommunitySettings, - is_valid_resource, - validate_operation, - validate_resource, - validate_resource_type, -) -from tribler.core.knowledge.payload import StatementOperation, StatementOperationMessage - -if TYPE_CHECKING: - from ipv8.community import CommunitySettings - from ipv8.test.mocking.ipv8 import MockIPv8 - - -@dataclasses.dataclass -class Peer: - """ - A mocked Peer class. - """ - - public_key: bytes - added_at: datetime = dataclasses.field(default=datetime.now(tz=timezone.utc)) - operations: set[StatementOp] = dataclasses.field(default_factory=set) - - -@dataclasses.dataclass -class Resource: - """ - A mocked Resource class. - """ - - name: str - type: int - - subject_statements: set[Statement] = dataclasses.field(default_factory=set) - object_statements: set[Statement] = dataclasses.field(default_factory=set) - torrent_healths: set = dataclasses.field(default_factory=set) - trackers: set = dataclasses.field(default_factory=set) - - -@dataclasses.dataclass -class Statement: - """ - A mocked Statement class. - """ - - subject: Resource - object: Resource - operations: set[StatementOp] - added_count: int = 0 - removed_count: int = 0 - local_operation: int = 0 - - -@dataclasses.dataclass -class StatementOp: - """ - A mocked StatementOp class. - """ - - statement: Statement - peer: Peer - operation: int - clock: int - signature: bytes - updated_at: datetime = dataclasses.field(default=datetime.now(tz=timezone.utc)) - auto_generated: bool = False - - -class TestKnowledgeCommunity(TestBase[KnowledgeCommunity]): - """ - Tests for the KnowledgeCommunity. - """ - - def setUp(self) -> None: - """ - Create two nodes. - """ - super().setUp() - self.initialize(KnowledgeCommunity, 2, - KnowledgeCommunitySettings(request_interval=0.1)) - self.operations = [] - self.signatures = [] - self.statement_ops = [] - - def create_node(self, settings: CommunitySettings | None = None, create_dht: bool = False, - enable_statistics: bool = False) -> MockIPv8: - """ - Create a mocked database and new key for each node. - """ - settings.db = Mock() - settings.key = default_eccrypto.generate_key("curve25519") - out = super().create_node(settings, create_dht, enable_statistics) - out.overlay.cancel_all_pending_tasks() - return out - - def create_operation(self, subject: str = "1" * 20, obj: str = "", - sign_correctly: bool = True) -> StatementOperation: - """ - Create an operation with the given subject and object. - """ - operation = StatementOperation(subject_type=ResourceType.TORRENT, subject=subject, predicate=ResourceType.TAG, - object=obj, operation=Operation.ADD, clock=len(self.operations) + 1, - creator_public_key=self.overlay(0).key.pub().key_to_bin()) - self.operations.append((operation, self.overlay(0).sign(operation) if sign_correctly else b'1' * 64)) - self.statement_ops.append(StatementOp( - statement=Statement( - Resource(self.operations[-1][0].subject, self.operations[-1][0].subject_type), - Resource(self.operations[-1][0].object, self.operations[-1][0].predicate), - set() - ), - peer=Peer(self.overlay(0).key.pub().key_to_bin()), - operation=self.operations[-1][0].operation, - clock=self.operations[-1][0].clock, - signature=self.operations[-1][1] - )) - return operation - - @db_session - def fill_db(self) -> None: - """ - Create 5 correct, of which one has unicode characters, and 5 incorrect operations. - """ - for i in range(9): - self.create_operation(obj=f'{i}' * 3, sign_correctly=i < 4) - self.create_operation(subject='Контент', obj='Тэг', sign_correctly=True) - self.overlay(0).db.knowledge.get_operations_for_gossip = lambda count: sample(self.statement_ops, count) - - async def test_gossip(self) -> None: - """ - Test if the 5 correct messages are gossiped. - """ - self.fill_db() - - with self.assertReceivedBy(1, [StatementOperationMessage] * 10) as received: - self.overlay(1).request_operations() - await self.deliver_messages() - - received_objects = {message.operation.object for message in received} - self.assertEqual(10, len(received_objects)) - self.assertEqual(5, len(self.overlay(1).db.knowledge.add_operation.call_args_list)) - - async def test_on_request_eat_exceptions(self) -> None: - """ - Test faulty statement ops have their ValueError caught. - """ - self.fill_db() - self.statement_ops[0].statement.subject.name = "" # Fails validate_resource(operation.subject) - - with self.assertReceivedBy(1, [StatementOperationMessage] * 9) as received: - self.overlay(1).request_operations() - await self.deliver_messages() - - received_objects = {message.operation.object for message in received} - self.assertEqual(9, len(received_objects)) - self.assertEqual(4, len(self.overlay(1).db.knowledge.add_operation.call_args_list)) - - async def test_no_peers(self) -> None: - """ - Test if no error occurs in the community, in case there are no peers. - """ - self.overlay(1).network.remove_peer(self.peer(0)) - self.fill_db() - - with self.assertReceivedBy(0, []), self.assertReceivedBy(1, []): - self.overlay(1).request_operations() - await self.deliver_messages() - - def test_valid_tag(self) -> None: - """ - Test if a normal tag is valid. - """ - tag = "Tar " - - validate_resource(tag) # no exception - - self.assertTrue(is_valid_resource(tag)) - - def test_invalid_tag_nothing(self) -> None: - """ - Test if nothing is not a valid tag. - """ - tag = "" - - self.assertFalse(is_valid_resource(tag)) - - with self.assertRaises(ValueError): - validate_resource(tag) - - def test_invalid_tag_short(self) -> None: - """ - Test if a short tag is not valid. - """ - tag = "t" - - self.assertFalse(is_valid_resource(tag)) - - with self.assertRaises(ValueError): - validate_resource(tag) - - def test_invalid_tag_long(self) -> None: - """ - Test if a long tag is not valid. - """ - tag = "t" * 51 - - self.assertFalse(is_valid_resource(tag)) - - with self.assertRaises(ValueError): - validate_resource(tag) - - def test_correct_operation(self) -> None: - """ - Test if a correct operation is valid. - """ - for operation in Operation: - validate_operation(operation) # no exception - validate_operation(operation.value) # no exception - - def test_incorrect_operation(self) -> None: - """ - Test if an incorrect operation raises a ValueError. - """ - max_operation = max(Operation) - - with self.assertRaises(ValueError): - validate_operation(max_operation.value + 1) - - def test_correct_relation(self) -> None: - """ - Test if a correct relation is valid. - """ - for relation in ResourceType: - validate_resource_type(relation) # no exception - validate_resource_type(relation.value) # no exception - - def test_incorrect_relation(self) -> None: - """ - Test if an incorrect relation raises a ValueError. - """ - max_relation = max(ResourceType) - with self.assertRaises(ValueError): - validate_operation(max_relation.value + 1) diff --git a/src/tribler/test_unit/core/knowledge/test_content_bundling.py b/src/tribler/test_unit/core/knowledge/test_content_bundling.py deleted file mode 100644 index a20ffc7eac..0000000000 --- a/src/tribler/test_unit/core/knowledge/test_content_bundling.py +++ /dev/null @@ -1,143 +0,0 @@ -from ipv8.test.base import TestBase - -from tribler.core.knowledge.content_bundling import _create_name, calculate_diversity, group_content_by_number - - -class TestContentBundling(TestBase): - """ - Tests for content bundling functionality. - """ - - def test_group_content_by_number_empty_list(self) -> None: - """ - Test if group_content_by_number returns an empty dict if an empty list passed. - """ - self.assertEqual({}, group_content_by_number([])) - - def test_group_content_by_number(self) -> None: - """ - Test if group_content_by_number group content by a first number. - """ - content_list = [ - {"name": "item 2"}, - {"name": "item 1"}, - {"name": "item with number1"}, - {"name": "item with number 2 and 3"}, - {"name": "item without number"}, - {"item": "without a name"}, - {"item": "without a name but with 1 number"}, - ] - - actual = group_content_by_number(content_list) - expected = { - "Item 2": [{"name": "item 2"}, {"name": "item with number 2 and 3"}], - "Item 1": [{"name": "item 1"}, {"name": "item with number1"}] - } - self.assertEqual(expected, actual) - - def test_group_content_by_number_extract_no_spaces(self) -> None: - """ - Test if group_content_by_number extracts correct group name from text without spaces. - """ - actual = group_content_by_number([{"name": "text123"}], min_group_size=1) - - self.assertEqual({"Text 123": [{"name": "text123"}]}, actual) - - def test_group_content_by_number_extract_period(self) -> None: - """ - Test if group_content_by_number extracts correct group name from text with a period. - """ - actual = group_content_by_number([{"name": "text.123"}], min_group_size=1) - - self.assertEqual({"Text 123": [{"name": "text.123"}]}, actual) - - def test_group_content_by_number_extract_complex(self) -> None: - """ - Test if group_content_by_number extracts correct group name from text with many numbers and strings. - """ - actual = group_content_by_number([{"name": "123any345text678"}], min_group_size=1) - - self.assertEqual({"Text 123": [{"name": "123any345text678"}]}, actual) - - def test_group_content_by_number_extract_simplify_number(self) -> None: - """ - Test if group_content_by_number extracts correct group name from text with a 0-prepended number. - """ - actual = group_content_by_number([{"name": "012"}], min_group_size=1) - - self.assertEqual({"12": [{"name": "012"}]}, actual) - - def test_create_name(self) -> None: - """ - Test if _create_name creates a group name based on the most common word in the title. - """ - content_list = [ - {"name": "Individuals and interactions over processes and tools"}, - {"name": "Working software over comprehensive documentation"}, - {"name": "Customer collaboration over contract negotiation"}, - {"name": "Responding to change over following a plan"}, - ] - - self.assertEqual("Over 1", _create_name(content_list, "1", min_word_length=4)) - - def test_create_name_non_latin(self) -> None: - """ - Test if _create_name creates a group name based on the most common word in the title with non-latin characters. - """ - content_list = [ - {"name": "Может быть величайшим триумфом человеческого гения является то, "}, - {"name": "что человек может понять вещи, которые он уже не в силах вообразить"}, - ] - - self.assertEqual("Может 2", _create_name(content_list, "2")) - - def test_calculate_diversity_match_one(self) -> None: - """ - Test if calculate_diversity finds one other word and calculates the CTTR. - """ - content_list = [{"name": "word wor wo w"}] - - self.assertEqual(10, int(10.0 * calculate_diversity(content_list, 3))) - - def test_calculate_diversity_match_two(self) -> None: - """ - Test if calculate_diversity finds two other words and calculates the CTTR. - """ - content_list = [{"name": "word wor wo w"}] - - self.assertEqual(12, int(10.0 * calculate_diversity(content_list, 2))) - - def test_calculate_diversity_match_three(self) -> None: - """ - Test if calculate_diversity finds three other words and calculates the CTTR. - """ - content_list = [{"name": "word wor wo w"}] - - self.assertEqual(14, int(10.0 * calculate_diversity(content_list, 1))) - - def test_calculate_diversity_match_all(self) -> None: - """ - Test if calculate_diversity finds all (three) other words and calculates the CTTR. - """ - content_list = [{"name": "word wor wo w"}] - - self.assertEqual(14, int(10.0 * calculate_diversity(content_list, 1))) - - def test_calculate_diversity_no_words(self) -> None: - """ - Test if calculate_diversity returns 0 if there are no words in the content list. - """ - content_list = [{"name": ""}] - - self.assertEqual(0, calculate_diversity(content_list)) - - def test_calculate_diversity(self) -> None: - """ - Test if calculate_diversity calculates diversity based on the text. - """ - self.assertEqual(70, int(100.0 * calculate_diversity([{"name": "The"}], min_word_length=3))) - self.assertEqual(100, int(100.0 * calculate_diversity([{"name": "The quick"}], min_word_length=3))) - self.assertEqual(122, int(100.0 * calculate_diversity([{"name": "The quick brown"}], min_word_length=3))) - self.assertEqual(106, int(100.0 * calculate_diversity([{"name": "The quick brown the"}], min_word_length=3))) - self.assertEqual(94, int(100.0 * calculate_diversity([{"name": "The quick brown the quick"}], - min_word_length=3))) diff --git a/src/tribler/test_unit/core/knowledge/test_operations_requests.py b/src/tribler/test_unit/core/knowledge/test_operations_requests.py deleted file mode 100644 index 8bb000943b..0000000000 --- a/src/tribler/test_unit/core/knowledge/test_operations_requests.py +++ /dev/null @@ -1,61 +0,0 @@ -from ipv8.test.base import TestBase - -from tribler.core.knowledge.operations_requests import OperationsRequests - - -class TestOperationsRequests(TestBase): - """ - Tests for the OperationsRequests class. - """ - - def setUp(self) -> None: - """ - Create a new OperationsRequests to test with. - """ - super().setUp() - self.operations_requests = OperationsRequests() - - def test_add_peer(self) -> None: - """ - Test if a peer can be registered. - """ - self.operations_requests.register_peer("peer", number_of_responses=10) - - self.assertEqual(10, self.operations_requests.requests["peer"]) - - def test_clear_requests(self) -> None: - """ - Test if requests can be cleared. - """ - self.operations_requests.register_peer("peer", number_of_responses=10) - - self.operations_requests.clear_requests() - - self.assertEqual(0, len(self.operations_requests.requests)) - - def test_valid_peer(self) -> None: - """ - Test if peers with a non-zero number of requests are seen as valid. - """ - self.operations_requests.register_peer("peer", number_of_responses=10) - - self.operations_requests.validate_peer("peer") - - self.assertIn("peer", self.operations_requests.requests) - - def test_missed_peer(self) -> None: - """ - Test if peers with zero requests are seen as invalid. - """ - with self.assertRaises(ValueError): - self.operations_requests.validate_peer("peer") - - def test_invalid_peer(self) -> None: - """ - Test if validating a peer lowers it response count. - """ - self.operations_requests.register_peer("peer", number_of_responses=1) - self.operations_requests.validate_peer("peer") - - with self.assertRaises(ValueError): - self.operations_requests.validate_peer("peer") diff --git a/src/tribler/test_unit/core/knowledge/test_payload.py b/src/tribler/test_unit/core/knowledge/test_payload.py deleted file mode 100644 index c4e4278d16..0000000000 --- a/src/tribler/test_unit/core/knowledge/test_payload.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -from ipv8.test.base import TestBase - -from tribler.core.knowledge.payload import ( - RawStatementOperationMessage, - RequestStatementOperationMessage, - StatementOperation, - StatementOperationMessage, - StatementOperationSignature, -) - - -class TestHTTPPayloads(TestBase): - """ - Tests for the various payloads of the KnowledgeCommunity. - """ - - def test_statement_operation(self) -> None: - """ - Test if StatementOperation initializes correctly. - """ - so = StatementOperation(1, "foo", 2, "bar", 3, 4, b"baz") - - self.assertEqual(1, so.subject_type) - self.assertEqual("foo", so.subject) - self.assertEqual(2, so.predicate) - self.assertEqual("bar", so.object) - self.assertEqual(3, so.operation) - self.assertEqual(4, so.clock) - self.assertEqual(b"baz", so.creator_public_key) - - def test_statement_operation_signature(self) -> None: - """ - Test if StatementOperationSignature initializes correctly. - """ - sos = StatementOperationSignature(b"test") - - self.assertEqual(b"test", sos.signature) - - def test_raw_statement_operation_message(self) -> None: - """ - Test if RawStatementOperationMessage initializes correctly. - """ - rsom = RawStatementOperationMessage(b"foo", b"bar") - - self.assertEqual(2, rsom.msg_id) - self.assertEqual(b"foo", rsom.operation) - self.assertEqual(b"bar", rsom.signature) - - def test_statement_operation_message(self) -> None: - """ - Test if StatementOperationMessage initializes correctly. - """ - som = StatementOperationMessage(b"foo", b"bar") - - self.assertEqual(2, som.msg_id) - self.assertEqual(b"foo", som.operation) - self.assertEqual(b"bar", som.signature) - - def test_request_statement_operation_message(self) -> None: - """ - Test if RequestStatementOperationMessage initializes correctly. - """ - rsom = RequestStatementOperationMessage(42) - - self.assertEqual(1, rsom.msg_id) - self.assertEqual(42, rsom.count) diff --git a/src/tribler/tribler_config.py b/src/tribler/tribler_config.py index fdf67cfa14..88c0e1cba8 100644 --- a/src/tribler/tribler_config.py +++ b/src/tribler/tribler_config.py @@ -47,14 +47,6 @@ class DHTDiscoveryCommunityConfig(TypedDict): enabled: bool -class KnowledgeCommunityConfig(TypedDict): - """ - Settings for the knowledge component. - """ - - enabled: bool - - class DatabaseConfig(TypedDict): """ Settings for the database component. @@ -169,7 +161,6 @@ class TriblerConfig(TypedDict): content_discovery_community: ContentDiscoveryCommunityConfig database: DatabaseConfig - knowledge_community: KnowledgeCommunityConfig libtorrent: LibtorrentConfig recommender: RecommenderConfig rendezvous: RendezvousConfig @@ -201,7 +192,6 @@ class TriblerConfig(TypedDict): "content_discovery_community": ContentDiscoveryCommunityConfig(enabled=True), "database": DatabaseConfig(enabled=True), "dht_discovery": DHTDiscoveryCommunityConfig(enabled=True), - "knowledge_community": KnowledgeCommunityConfig(enabled=True), "libtorrent": LibtorrentConfig( socks_listen_ports=[0, 0, 0, 0, 0], port=0, diff --git a/src/tribler/ui/src/models/settings.model.tsx b/src/tribler/ui/src/models/settings.model.tsx index f65a930541..ad3b29422a 100644 --- a/src/tribler/ui/src/models/settings.model.tsx +++ b/src/tribler/ui/src/models/settings.model.tsx @@ -48,9 +48,6 @@ export interface Settings { dht_discovery: { enabled: boolean; }, - knowledge_community: { - enabled: boolean; - }, libtorrent: { socks_listen_ports: number[]; port: number; diff --git a/src/tribler/upgrade_script.py b/src/tribler/upgrade_script.py index 0fe276204a..4492db4b13 100644 --- a/src/tribler/upgrade_script.py +++ b/src/tribler/upgrade_script.py @@ -180,36 +180,6 @@ def _inject_StatementOp_inner(db: Database, batch: list) -> None: globals(), locals()) -def _inject_StatementOp(abs_src_db: str, abs_dst_db: str) -> None: - """ - Import old StatementOp entries. - """ - src_con = sqlite3.connect(abs_src_db) - output = list(src_con.execute("""SELECT SubjectResource.name, SubjectResource.type, ObjectResource.name, -ObjectResource.type, Statement.added_count, Statement.removed_count, Statement.local_operation, Peer.public_key, -Peer.added_at, StatementOp.operation, StatementOp.clock, StatementOp.signature, StatementOp.updated_at, -StatementOp.auto_generated -FROM StatementOp -INNER JOIN Peer ON StatementOp.peer=Peer.id -INNER JOIN Statement ON StatementOp.statement=Statement.id -INNER JOIN Resource AS SubjectResource ON Statement.subject=SubjectResource.id -INNER JOIN Resource AS ObjectResource ON Statement.object=ObjectResource.id -;""")) - src_con.close() - - db = Database() - db.bind(provider="sqlite", filename=abs_dst_db) - for batch in batched(output, n=20): - try: - _inject_StatementOp_inner(db, batch) - except OperationalError as e: - logging.exception(e) - try: - db.disconnect() - except OperationalError as e: - logging.exception(e) - - @db_session(retry=3) def _inject_ChannelNode_inner(db: Database, batch: list) -> None: """ @@ -370,7 +340,7 @@ def _inject_TorrentState_TrackerState(abs_src_db: str, abs_dst_db: str) -> None: logging.exception(e) -def _inject_7_14_tables(src_db: str, dst_db: str, db_format: str) -> None: +def _inject_7_14_tables(src_db: str, dst_db: str) -> None: """ Fetch data from the old database and attempt to insert it into a new one. """ @@ -383,18 +353,12 @@ def _inject_7_14_tables(src_db: str, dst_db: str, db_format: str) -> None: shutil.copy(src_db, dst_db) return - # If they both exist, we have to inject data. - assert db_format in ["tribler.db", "metadata.db"] - abs_src_db = os.path.abspath(src_db) abs_dst_db = os.path.abspath(dst_db) - if db_format == "tribler.db": - _inject_StatementOp(abs_src_db, abs_dst_db) - else: - _inject_ChannelNode(abs_src_db, abs_dst_db) - _inject_TrackerState(abs_src_db, abs_dst_db) - _inject_TorrentState_TrackerState(abs_src_db, abs_dst_db) + _inject_ChannelNode(abs_src_db, abs_dst_db) + _inject_TrackerState(abs_src_db, abs_dst_db) + _inject_TorrentState_TrackerState(abs_src_db, abs_dst_db) def upgrade(config: TriblerConfigManager, source: str, destination: str) -> None: @@ -420,17 +384,10 @@ def upgrade(config: TriblerConfigManager, source: str, destination: str) -> None _copy_if_not_exist(os.path.join(source, "dlcheckpoints", checkpoint), os.path.join(parent_directory, "dlcheckpoints", checkpoint)) - # Step 3: Copy tribler db. - os.makedirs(os.path.join(destination, "sqlite"), exist_ok=True) - _inject_7_14_tables(os.path.join(source, "sqlite", "tribler.db"), - os.path.join(destination, "sqlite", "tribler.db"), - "tribler.db") - - # Step 4: Copy metadata db. + # Step 3: Copy metadata db. _inject_7_14_tables(os.path.join(source, "sqlite", "metadata.db"), - os.path.join(destination, "sqlite", "metadata.db"), - "metadata.db") + os.path.join(destination, "sqlite", "metadata.db")) - # Step 5: Signal that our upgrade is done. + # Step 4: Signal that our upgrade is done. with open(os.path.join(config.get_version_state_dir(), ".upgraded"), "a"): pass