Merge pull request #38 from nickpadd/develop

Important update to sqlite3

europeanfootballleaguepredictor/common/config_parser.py

@@ -92,9 +92,7 @@ def load_configuration_class(self, config_data) -> dataclass:
                 seasons_to_gather= config_data['data_gathering']['seasons_to_gather'],
                 current_season= config_data['data_gathering']['current_season'],
                 data_co_uk_path= config_data['data_gathering']['paths'][config_data['league']]['data_co_uk_path'],
-                raw_data_path= config_data['data_gathering']['paths'][config_data['league']]['raw_data_path'],
-                preprocessed_data_path= config_data['data_gathering']['paths'][config_data['league']]['preprocessed_data_path'],
-                fixture_download_path= config_data['data_gathering']['paths'][config_data['league']]['fixture_download'],
+                database = config_data['data_gathering']['paths'][config_data['league']]['database'],
                 bookmaker_url= config_data['data_gathering']['bookmaker'][config_data['league']]['url'],
                 bookmaker_dictionary= self.load_and_extract_yaml_section(path = config_data['data_gathering']['bookmaker'][config_data['league']]['dictionary_path']),
                 data_co_uk_url= config_data['data_gathering']['data_co_uk'][config_data['league']]['url'],
@@ -119,12 +117,10 @@ class Configuration:
     bettor_kelly_cap: float
     evaluation_output: str
     months_of_form_list: list
+    database: str
     seasons_to_gather: list
     current_season: str
     data_co_uk_path: str
-    raw_data_path: str
-    preprocessed_data_path: str
-    fixture_download_path: str
     bookmaker_url: str
     bookmaker_dictionary: dict
     data_co_uk_url: str

europeanfootballleaguepredictor/config/config.yaml

@@ -14,39 +14,29 @@ bettor:
 data_gathering:
   long_term_form: null #Months of long_form, null indicates season long form
   short_term_form: 3 #Months of short-term form
-  seasons_to_gather: ['2017', '2018', '2019', '2020', '2021', '2022', '2023'] #List of the seasons to gather for the model to be trained on. Keep in mind that 2020 season was covid-season
+  seasons_to_gather: ['2017', '2018', '2019', '2020', '2021', '2022', '2023'] #List of the seasons to gather for the model to be trained on. Subset of ['2017', '2018', '2019', '2020', '2021', '2022', '2023']. Keep in mind that 2020 season was covid-season
   current_season: '2023'
   paths:
     EPL: 
-      data_co_uk_path: 'europeanfootballleaguepredictor/data/leagues/EPL/season/DataCoUkFiles/'
-      raw_data_path: 'europeanfootballleaguepredictor/data/leagues/EPL/season/raw_files/'
-      preprocessed_data_path: 'europeanfootballleaguepredictor/data/leagues/EPL/season/preprocessed_files/'
-      fixture_download: 'europeanfootballleaguepredictor/data/leagues/EPL/upcoming/'
+      data_co_uk_path: 'europeanfootballleaguepredictor/data/leagues/EPL/DataCoUkFiles/'
       evaluation_output: 'europeanfootballleaguepredictor/data/leagues/EPL/evaluation/'
+      database: 'europeanfootballleaguepredictor/data/database/EPL_database.db'
     La_Liga:
-      data_co_uk_path: 'europeanfootballleaguepredictor/data/leagues/La_Liga/season/DataCoUkFiles/'
-      raw_data_path: 'europeanfootballleaguepredictor/data/leagues/La_Liga/season/raw_files/'
-      preprocessed_data_path: 'europeanfootballleaguepredictor/data/leagues/La_Liga/season/preprocessed_files/'
-      fixture_download: 'europeanfootballleaguepredictor/data/leagues/La_Liga/upcoming/'
+      data_co_uk_path: 'europeanfootballleaguepredictor/data/leagues/La_Liga/DataCoUkFiles/'
       evaluation_output: 'europeanfootballleaguepredictor/data/leagues/La_Liga/evaluation/'
+      database: 'europeanfootballleaguepredictor/data/database/La_Liga_database.db'
     Bundesliga:
-      data_co_uk_path: 'europeanfootballleaguepredictor/data/leagues/Bundesliga/season/DataCoUkFiles/'
-      raw_data_path: 'europeanfootballleaguepredictor/data/leagues/Bundesliga/season/raw_files/'
-      preprocessed_data_path: 'europeanfootballleaguepredictor/data/leagues/Bundesliga/season/preprocessed_files/'
-      fixture_download: 'europeanfootballleaguepredictor/data/leagues/Bundesliga/upcoming/' 
+      data_co_uk_path: 'europeanfootballleaguepredictor/data/leagues/Bundesliga/DataCoUkFiles/'
       evaluation_output: 'europeanfootballleaguepredictor/data/leagues/Bundesliga/evaluation/'
+      database: 'europeanfootballleaguepredictor/data/database/Bundesliga_database.db'
     Serie_A:
-      data_co_uk_path: 'europeanfootballleaguepredictor/data/leagues/Serie_A/season/DataCoUkFiles/'
-      raw_data_path: 'europeanfootballleaguepredictor/data/leagues/Serie_A/season/raw_files/'
-      preprocessed_data_path: 'europeanfootballleaguepredictor/data/leagues/Serie_A/season/preprocessed_files/'
-      fixture_download: 'europeanfootballleaguepredictor/data/leagues/Serie_A/upcoming/' 
+      data_co_uk_path: 'europeanfootballleaguepredictor/data/leagues/Serie_A/DataCoUkFiles/'
       evaluation_output: 'europeanfootballleaguepredictor/data/leagues/Serie_A/evaluation/'
+      database: 'europeanfootballleaguepredictor/data/database/Serie_A_database.db'
     Ligue_1:
-      data_co_uk_path: 'europeanfootballleaguepredictor/data/leagues/Ligue_1/season/DataCoUkFiles/'
-      raw_data_path: 'europeanfootballleaguepredictor/data/leagues/Ligue_1/season/raw_files/'
-      preprocessed_data_path: 'europeanfootballleaguepredictor/data/leagues/Ligue_1/season/preprocessed_files/'
-      fixture_download: 'europeanfootballleaguepredictor/data/leagues/Ligue_1/upcoming/'      
+      data_co_uk_path: 'europeanfootballleaguepredictor/data/leagues/Ligue_1/DataCoUkFiles/'
       evaluation_output: 'europeanfootballleaguepredictor/data/leagues/Ligue_1/evaluation/'
+      database: 'europeanfootballleaguepredictor/data/database/Ligue_1_database.db'
   bookmaker: 
     EPL: 
       url: 'https://en.stoiximan.gr/sport/soccer/england/premier-league/1/' #The corresponding stoiximan website url of the predicted league. Might not work in countries outside Greece.

europeanfootballleaguepredictor/data/bookmaker_scraper.py

@@ -7,23 +7,22 @@
 from loguru import logger
 
 class BookmakerScraper():
-    """A class responsible for scraping the bookmaker website
-    """
+    """A class responsible for scraping the bookmaker website"""
     def __init__(self, url: str, dictionary: dict):
-        """Initializing the scraper by specifying the url and the dictionary of the team names used by the bookmaker
+        """Initializing the scraper by specifying the url and the dictionary of the team names used by the bookmaker.
 
         Args:
-            url (str): The url corresponding to the certain webpage with the betting odds of the league specified in the configuration
-            dictionary (dict): A dictionary of the team names used by the bookmaker
+            url (str): The url corresponding to the certain webpage with the betting odds of the league specified in the configuration.
+            dictionary (dict): A dictionary of the team names used by the bookmaker.
         """
         self.url = url
         self.dictionary = dictionary
 
     def get_odds_json(self) -> dict:
-        """Gets a page dictionary containing the odds, from the specified url
+        """Gets a page dictionary containing the odds, from the specified url.
 
         Returns:
-            dict: A dictionary containing the odds together with other raw code elements
+            dict: A dictionary containing the odds together with other raw code elements.
         """
         page = requests.get(self.url)
         soup = BeautifulSoup(page.content, "html.parser")
@@ -79,7 +78,6 @@ def odds_json_to_dataframe(self, odds_dictionary: dict) -> pd.DataFrame:
                 names = names[7:]
 
         data_teams = pd.DataFrame({'Home Team': HomeTeams, 'Away Team': AwayTeams})
-        logger.debug(line)
         data_values = pd.DataFrame(rows, columns=['1', 'x', '2', 'Line', 'OverLineOdds', 'UnderLineOdds', 'Yes', 'No'])
         odds_dataframe = pd.concat([data_teams, data_values], axis=1)
         odds_dataframe['Home Team'] = odds_dataframe['Home Team'].str.strip()

europeanfootballleaguepredictor/data/database_handler.py

@@ -0,0 +1,104 @@
+import sqlite3 
+import pandas as pd 
+from loguru import logger 
+import sys
+from sqlalchemy import TEXT
+
+class DatabaseHandler:
+    """A class for handling SQLite databases and interacting with dataframes."""
+    def __init__(self, league:str, database: str):
+        """
+        Initializes the DatabaseHandler.
+
+        Args:
+            league (str): The name of the league associated with the database.
+            database (str): The path to the SQLite database file.
+        """
+        self.league = league
+        self.database = database
+
+    def get_data(self, table_names: str or list[str]) -> list[pd.DataFrame]:
+        """
+        Retrieves data from the specified tables in the SQLite database.
+
+        Args:
+            table_names (str or list): A single table name or a list of table names to fetch data from.
+
+        Returns:
+            list[pd.DataFrame]: A list of dataframes containing the fetched data.
+        """
+        dataframe_list = []
+        connection = None
+
+        if isinstance(table_names, str):
+            table_names = [table_names]
+
+        try:
+            # Connect to the SQLite database
+            connection = sqlite3.connect(self.database)
+            cursor = connection.cursor()
+
+            for table_name in table_names:
+                # Check if the table exists
+                cursor.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name=?;", (table_name,))
+                result = cursor.fetchone()
+
+                if result:
+                    # Fetch data from the specified table
+                    query = f"SELECT * FROM {table_name};"
+                    dataframe_list.append(pd.read_sql_query(query, connection))
+                    logger.info(f"Data fetched for table: {table_name}")
+                else:
+                    logger.warning(f"Table '{table_name}' does not exist in the database. Skipping.")
+
+            return dataframe_list
+
+        except sqlite3.Error as e:
+            logger.error(f"SQLite error: {e}")
+
+        finally:
+            # Close the database connection
+            if connection:
+                connection.close()       
+
+    def save_dataframes(self, dataframes: list, table_names: list):
+        """
+        Saves dataframes to the corresponding tables in the SQLite database.
+
+        Args:
+            dataframes (list): A list of dataframes to be saved.
+            table_names (list): A list of table names corresponding to the dataframes.
+        """
+        if isinstance(table_names, str):
+            table_names = [table_names]
+        if isinstance(dataframes, pd.DataFrame):
+            dataframes = [dataframes]
+
+        if len(dataframes) != len(table_names):
+            logger.error("Length of dataframe_list must be equal to the length of table_names.")
+            sys.exit(1)
+
+        try:
+            # Connect to the SQLite database
+            connection = sqlite3.connect(self.database)
+            cursor = connection.cursor()
+
+            for df, table_name in zip(dataframes, table_names):
+                try:
+                    # Save the DataFrame to the corresponding table in the database
+                    df.to_sql(table_name, connection, index=False, if_exists='replace')
+                    logger.info(f'Table {table_name} created/updated for {self.league} league.')
+
+                except Exception as e:
+                    # Print or log the DataFrame to identify the issue
+                    logger.debug(f"DataFrame for {table_name}:\n{df.head()}")
+                    logger.debug(f"Error saving DataFrame to table {table_name}: {e}")
+
+        except sqlite3.Error as e:
+            print(f"SQLite error: {e}")
+
+        finally:
+            connection.commit()
+            # Close the database connection
+            if connection:
+                connection.close()