diff --git a/parser_wc.py b/parser_wc.py new file mode 100644 index 0000000..f9dda23 --- /dev/null +++ b/parser_wc.py @@ -0,0 +1,387 @@ +import time +from collections import defaultdict + +import logs_calendar +import logs_main +import top_gear +from api_db import DB, Table +from api_top_db_v2 import ( + TableTop, + TopDB, + Columns as ColumnsTop, +) +from api_wow_circle import ( + WCCharactersProfiles, + WCServer, +) +from c_path import Directories, StrEnum +from h_debug import Loggers +from h_server_fix import SERVERS_WC + +LOGGER = Loggers.gear + +SERVER_NAME = "WoW-Circle" +BOSSES_ICC = [ + "Lord Marrowgar", + "Lady Deathwhisper", + "Deathbringer Saurfang", + "Festergut", + "Rotface", + "Professor Putricide", + "Blood Prince Council", + "Blood-Queen Lana'thel", + "Sindragosa", + "The Lich King", +] + + +class Columns(StrEnum): + REPORT_ID = "report_id" + SERVER = "server" + +class ColumnsPlayers(StrEnum): + GUID = "guid" + NAME = "name" + + +class TableTopExt(TableTop): + def query_full_row_by_report(self, report_id_no_server: str): + return f''' + SELECT * + FROM [{self.name}] + WHERE {ColumnsTop.REPORT_ID}='{report_id_no_server}' + ''' + +class WCLogsTable(Table): + without_row_id = True + COLUMNS_ORDERED = list(Columns) + COLUMNS_TABLE_CREATE = [ + f"{COLUMNS_ORDERED[0]} PRIMARY KEY", + *COLUMNS_ORDERED[1:], + ] + def __init__(self, name="main"): + super().__init__(name) + + def query_reports_servers(self, reports: list[str]): + reports_str = ','.join(f'"{x}"' for x in reports) + return '\n'.join(( + f"SELECT *", + f"FROM [{self.name}]", + f"WHERE {Columns.REPORT_ID} in ({reports_str})", + )) + + def query_reports_ids(self): + return '\n'.join(( + f"SELECT {Columns.REPORT_ID}", + f"FROM [{self.name}]", + )) + +class WCLogsPlayersTable(Table): + without_row_id = True + COLUMNS_ORDERED = list(ColumnsPlayers) + COLUMNS_TABLE_CREATE = [ + f"{COLUMNS_ORDERED[0]} PRIMARY KEY", + *COLUMNS_ORDERED[1:], + ] + def __init__(self, name: str): + super().__init__(name) + + def query_players(self, player_guids: list[int]): + player_guids_str = ','.join(map(str, player_guids)) + return '\n'.join(( + f"SELECT *", + f"FROM [{self.name}]", + f"WHERE {ColumnsPlayers.GUID} in ({player_guids_str})", + )) + + +class WCLogsDB(DB): + def __init__(self, new=False): + dirpath = Directories.db.new_child("custom_server") + path = dirpath / f"{SERVER_NAME}.db" + super().__init__(path, new) + self.main_table = WCLogsTable() + + def get_reports(self, reports: list[str]): + query = self.main_table.query_reports_servers(reports) + try: + return self.cursor.execute(query) + except Exception: + return [] + + def get_missing_reports(self, reports: set[str]): + query = self.main_table.query_reports_ids() + try: + cached_reports = { + report_id + for report_id, in self.cursor.execute(query) + } + except Exception: + cached_reports = set() + + return reports - cached_reports + +class WCLogsPlayersDB(WCLogsDB): + def __init__(self, server: str, new=False): + super().__init__(new) + self.server = server + self.players_table = WCLogsPlayersTable(self.server) + + def get_reports(self, reports: list[str]): + query = self.main_table.query_reports_servers(reports) + try: + return self.cursor.execute(query) + except Exception: + return [] + + def get_missing_reports(self, reports: set[str]): + reports = set(reports) + query = self.main_table.query_reports_ids() + try: + cached_reports = { + report_id + for report_id, in self.cursor.execute(query) + } + return reports - cached_reports + except Exception: + return reports + + + def add_report(self, report_id: str): + rows = [ + (report_id, self.server), + ] + self.add_new_rows(self.main_table, rows) + + def get_players(self, players: list[str]) -> dict[int, str]: + query = self.players_table.query_players(players) + try: + data = self.cursor.execute(query) + return dict(data) + except Exception: + return {} + + def add_players(self, players: dict[str, str]): + rows = list(players.items()) + self.add_new_rows(self.players_table, rows) + + +class ReportPlayers: + server_cache: defaultdict[str, dict[int, str]] + def __init__(self, report_id: str, max_items=5): + self.report_id = report_id + self.max_items = max_items + + @property + def players(self): + try: + return self._players + except AttributeError: + self._players = self._get_names_guids_pairs() + return self._players + + def get_server(self): + if not self.players: + return + + server = self.get_server_from_db() + if not server: + server = self.get_server_from_api() + LOGGER.debug(f"{self.report_id:40} | {server}") + return server + + def some_players(self): + items = list(self.players.items()) + if self.max_items: + items = items[:self.max_items] + return dict(items) + + def get_players_from_db(self, server: str): + try: + return WCLogsPlayersDB(server).get_players(self.players) + except FileNotFoundError: + return {} + + def get_server_from_db(self): + half = len(self.players) // 2 + for server in SERVERS_WC: + LOGGER.debug(f"> Checking from db {server}") + overlap = 0 + cached_players = self.get_players_from_db(server) + for player_guid, player_name in self.players.items(): + cached_name = cached_players.get(player_guid) + if cached_name != player_name: + continue + overlap += 1 + if overlap > half: + return server + + def get_server_from_api(self): + players = self.some_players() + server = self._get_server_from_api(players) + if not server: + LOGGER.warning(f"! Server not found by small request") + server = self._get_server_from_api(self.players) + return server + + def _get_server_from_api(self, players): + half = len(players) // 2 + for server in SERVERS_WC: + LOGGER.debug(f"> Checking from api {server}") + overlap = 0 + characters = WCServer(server).get_multi_char_data(players) + for character in characters: + if not character.exists: + continue + if character.name != players.get(character.guid): + continue + overlap += 1 + if overlap > half: + return server + + def _get_names_guids_pairs(self): + report = logs_main.THE_LOGS(self.report_id) + guids = report.PLAYERS_GUIDS + # guids_json = Directories.logs / self.report_id / "PLAYERS_DATA.json" + # guids: dict[str, str] = guids_json.json_ignore_error() + return { + int(guid, 16): name + for guid, name in guids.items() + } + + +class WCParser(ReportPlayers): + @property + def server(self): + try: + return self._server + except AttributeError: + self._server = self._find_server() + return self._server + + def _find_server(self): + server = self.get_server() + if not server: + return + db = WCLogsPlayersDB(server, new=True) + db.add_players(self.players) + db.add_report(self.report_id) + return server + + def insert_from_top_db(self): + db_main = TopDB(SERVER_NAME) + db = TopDB(self.server, new=True) + + report_id_no_server = self.report_id.rsplit("--", 1)[0] + + for table_name in db_main.tables_names(): + t = TableTopExt(table_name) + q = t.query_full_row_by_report(report_id_no_server) + rows = db_main.cursor.execute(q) + db.add_new_rows(t, rows) + + +class GearParser: + def __init__(self, server: str, slice_size: int=50): + self.server = server + self.slice_size = slice_size + self.delay = slice_size // 50 + 1 + + def update_gear(self, players: list[int]): + LOGGER.debug(f"{self.server:15} | update_gear length: {len(players)}") + players = list(players) + for i in range(0, len(players), self.slice_size): + LOGGER.debug(f"{self.server:15} | update_gear {i:>4} | {len(players):>4}") + players_slice = players[i:i + self.slice_size] + self._update_gear(players_slice) + + LOGGER.debug(f"Sleeping for {self.delay} sec") + time.sleep(self.delay) + + def _update_gear(self, players: list[int]): + wc_chars = WCCharactersProfiles(self.server).get_profiles(players) + LOGGER.debug(f"{self.server:15} | update_gear returned {len(wc_chars):>4} / {len(players):>4}") + + profiles_dicts = {} + for wc_char in wc_chars: + profiles_dicts[wc_char.char.name] = wc_char.profile() + + db = top_gear.GearDB(self.server, new=True) + db.update_players_rows(profiles_dicts) + +###################### +def parse_specific(reports: list[str]): + players_groupped = defaultdict(dict) + + for report_id in reports: + p = WCParser(report_id) + if not p.server: + continue + p.insert_from_top_db() + players_groupped[p.server].update(p.players) + + for server, players in players_groupped.items(): + GearParser(server).update_gear(players) + +def gen_reports(): + df = logs_calendar.read_main_df() + df_filtered = df[df["server"] == SERVER_NAME] + z = set(df_filtered.index) + return z + +def parse_new_reports(): + db = WCLogsDB() + reports = gen_reports() + reports = db.get_missing_reports(reports) + print(reports) + parse_specific(reports) + +###################### +def players_from_top(server: str, boss_name: str): + db = TopDB(server) + q = "\n".join(( + f"SELECT guid", + f"FROM [{boss_name}.25H]", + )) + return { + int(f"0x{x}", 16) + for x, in db.cursor.execute(q) + } + +def server_players(server): + players = set() + for boss in BOSSES_ICC: + try: + players.update(players_from_top(server, boss)) + except Exception: + pass + return sorted(players) + +def parse_whole_top(): + for server in SERVERS_WC: + players = server_players(server) + GearParser(server).update_gear(players) + +def argv_get_report_name(flags: list[str]): + try: + i = flags.index("-i") + 1 + return flags[i] + except IndexError: + raise ValueError("-i flag must be paired with file name string") + +def main(): + import sys + if "-i" in sys.argv: + reports = [ + argv_get_report_name(sys.argv), + ] + parse_specific(reports) + return + + parse_new_reports() + +if __name__ == "__main__": + try: + main() + except Exception: + LOGGER.exception("parcer wc main")