emm_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 1: Environment Setup\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import psutil\n",
    "import gc\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "def create_project_dirs():\n",
    "    \"\"\"Create project directories if they don't exist\"\"\"\n",
    "    base_dirs = ['data', 'plots', 'animations', 'models', 'results']\n",
    "    for d in base_dirs:\n",
    "        os.makedirs(d, exist_ok=True)\n",
    "\n",
    "def get_memory_usage():\n",
    "    \"\"\"Return current memory usage in MB\"\"\"\n",
    "    return psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024\n",
    "\n",
    "def clear_memory():\n",
    "    \"\"\"Clear unused memory\"\"\"\n",
    "    gc.collect()\n",
    "    \n",
    "def set_plotting_style():\n",
    "    \"\"\"Set consistent plotting style\"\"\"\n",
    "    plt.style.use('ggplot')\n",
    "    plt.rcParams['figure.figsize'] = [10, 6]\n",
    "    plt.rcParams['figure.dpi'] = 100\n",
    "    plt.rcParams['font.size'] = 10\n",
    "    \n",
    "# NFC West teams\n",
    "NFC_WEST = ['ARI', 'LAR', 'SF', 'SEA']\n",
    "\n",
    "# Initialize environment\n",
    "print(f\"Initial memory usage: {get_memory_usage():.2f} MB\")\n",
    "create_project_dirs()\n",
    "set_plotting_style()\n",
    "print(\"\\nProject directories created:\")\n",
    "print(\"\\n\".join(os.listdir()))\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n",
    "\n",
    "# Verify pandas version and key settings\n",
    "print(\"\\nEnvironment Info:\")\n",
    "print(f\"Pandas version: {pd.__version__}\")\n",
    "print(f\"NumPy version: {np.__version__}\")\n",
    "pd.set_option('display.max_columns', None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 2: Data Loading & Filtering\n",
    "import os\n",
    "\n",
    "# Print current working directory for verification\n",
    "print(\"Current working directory:\", os.getcwd())\n",
    "\n",
    "# Update NFC West teams constant - using 'LA' instead of 'LAR'\n",
    "NFC_WEST = ['ARI', 'LA', 'SF', 'SEA']\n",
    "\n",
    "# Define data directory using relative path (up one level)\n",
    "DATA_DIR = os.path.join(os.path.dirname(os.getcwd()), 'data')\n",
    "print(f\"\\nUsing data directory: {DATA_DIR}\")\n",
    "\n",
    "\n",
    "def load_nfc_west_games(weeks_range=[1, 2, 3, 4, 5]):\n",
    "    \"\"\"Load and filter games for NFC West teams\"\"\"\n",
    "    print(\"Loading games data...\")\n",
    "    games_path = os.path.join(DATA_DIR, 'games.csv')\n",
    "    games_df = pd.read_csv(games_path)\n",
    "    \n",
    "    # Filter for NFC West games\n",
    "    nfc_west_mask = (games_df['homeTeamAbbr'].isin(NFC_WEST) | \n",
    "                     games_df['visitorTeamAbbr'].isin(NFC_WEST))\n",
    "    weeks_mask = games_df['week'].isin(weeks_range)\n",
    "    \n",
    "    filtered_games = games_df[nfc_west_mask & weeks_mask].copy()\n",
    "    \n",
    "    print(f\"\\nFound {len(filtered_games)} NFC West games in weeks {weeks_range}\")\n",
    "    return filtered_games[['gameId', 'week', 'homeTeamAbbr', 'visitorTeamAbbr']]\n",
    "\n",
    "def load_tracking_data(games_df, week):\n",
    "    \"\"\"Load tracking data for specific week with memory optimization\"\"\"\n",
    "    print(f\"\\nProcessing week {week}\")\n",
    "    print(f\"Memory before loading: {get_memory_usage():.2f} MB\")\n",
    "    \n",
    "    tracking_file = os.path.join(DATA_DIR, f'tracking_week_{week}.csv')\n",
    "    relevant_game_ids = set(games_df['gameId'])\n",
    "    \n",
    "    # Read with chunking for memory efficiency\n",
    "    chunks = []\n",
    "    chunk_size = 100000\n",
    "    \n",
    "    for chunk in tqdm(pd.read_csv(tracking_file, chunksize=chunk_size)):\n",
    "        # Filter for relevant games\n",
    "        filtered_chunk = chunk[chunk['gameId'].isin(relevant_game_ids)].copy()\n",
    "        chunks.append(filtered_chunk)\n",
    "        \n",
    "        # Clear memory\n",
    "        del filtered_chunk\n",
    "        clear_memory()\n",
    "        \n",
    "        # Check memory usage\n",
    "        if get_memory_usage() > 32000:  # 32GB warning\n",
    "            print(\"Warning: High memory usage detected\")\n",
    "    \n",
    "    # Combine chunks\n",
    "    week_data = pd.concat(chunks, ignore_index=True)\n",
    "    clear_memory()\n",
    "    \n",
    "    print(f\"Memory after loading: {get_memory_usage():.2f} MB\")\n",
    "    return week_data\n",
    "\n",
    "def save_processed_data(df, filename):\n",
    "    \"\"\"Save processed data to parquet format\"\"\"\n",
    "    output_path = os.path.join(DATA_DIR, filename)\n",
    "    df.to_parquet(output_path, index=False)\n",
    "    print(f\"Saved to {output_path}\")\n",
    "\n",
    "# Execute data loading\n",
    "print(f\"Starting memory usage: {get_memory_usage():.2f} MB\")\n",
    "\n",
    "# Load game data\n",
    "games = load_nfc_west_games()\n",
    "save_processed_data(games, 'nfc_west_games.parquet')\n",
    "\n",
    "# Process each week\n",
    "all_tracking_data = []\n",
    "for week in range(1, 6):\n",
    "    week_data = load_tracking_data(games, week)\n",
    "    \n",
    "    # Save week data\n",
    "    save_processed_data(week_data, f'nfc_west_week_{week}.parquet')\n",
    "    \n",
    "    # Keep summary stats\n",
    "    summary = {\n",
    "        'week': week,\n",
    "        'n_plays': week_data['playId'].nunique(),\n",
    "        'n_players': week_data['nflId'].nunique(),\n",
    "        'n_frames': len(week_data)\n",
    "    }\n",
    "    all_tracking_data.append(summary)\n",
    "    \n",
    "    # Clear memory\n",
    "    del week_data\n",
    "    clear_memory()\n",
    "\n",
    "# Create summary DataFrame\n",
    "summary_df = pd.DataFrame(all_tracking_data)\n",
    "print(\"\\nData Loading Summary:\")\n",
    "print(summary_df)\n",
    "\n",
    "# Save summary to results directory (up one level, then into results)\n",
    "RESULTS_DIR = os.path.join(os.path.dirname(os.getcwd()), 'results')\n",
    "os.makedirs(RESULTS_DIR, exist_ok=True)\n",
    "summary_df.to_csv(os.path.join(RESULTS_DIR, 'data_loading_summary.csv'), index=False)\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n",
    "\n",
    "# Display NFC West game distribution\n",
    "team_games = pd.concat([\n",
    "    games['homeTeamAbbr'].value_counts(),\n",
    "    games['visitorTeamAbbr'].value_counts()\n",
    "], axis=1)\n",
    "team_games.columns = ['Home Games', 'Away Games']\n",
    "team_games.fillna(0, inplace=True)\n",
    "team_games['Total Games'] = team_games.sum(axis=1)\n",
    "team_games = team_games.loc[NFC_WEST]\n",
    "\n",
    "print(\"\\nNFC West Game Distribution:\")\n",
    "print(team_games)\n",
    "team_games.to_csv(os.path.join(RESULTS_DIR, 'nfc_west_game_distribution.csv'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 3: Basic Movement Analysis\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "def load_players_data():\n",
    "    \"\"\"Load and filter for defensive players\"\"\"\n",
    "    players_path = os.path.join(DATA_DIR, 'players.csv')\n",
    "    players_df = pd.read_csv(players_path)\n",
    "    \n",
    "    # Define defensive positions\n",
    "    defensive_positions = ['SS', 'FS', 'CB', 'DB', 'S', 'ILB', 'LB', 'MLB', 'OLB']\n",
    "    return players_df[players_df['position'].isin(defensive_positions)]\n",
    "\n",
    "def calculate_movement_metrics(week_data, defensive_players):\n",
    "    \"\"\"Calculate basic movement metrics for defensive players\"\"\"\n",
    "    # Filter for defensive players\n",
    "    defensive_data = week_data[week_data['nflId'].isin(defensive_players['nflId'])]\n",
    "    \n",
    "    # Calculate metrics by player\n",
    "    metrics = []\n",
    "    for player_id, player_data in defensive_data.groupby('nflId'):\n",
    "        player_metrics = {\n",
    "            'nflId': player_id,\n",
    "            'position': defensive_players[defensive_players['nflId'] == player_id]['position'].iloc[0],\n",
    "            'team': player_data['club'].iloc[0],\n",
    "            'avg_speed': player_data['s'].mean(),\n",
    "            'max_speed': player_data['s'].max(),\n",
    "            'avg_acceleration': player_data['a'].mean(),\n",
    "            'direction_changes': np.sum(np.abs(np.diff(player_data['dir'])) > 10)\n",
    "        }\n",
    "        metrics.append(player_metrics)\n",
    "    \n",
    "    return pd.DataFrame(metrics)\n",
    "\n",
    "# Load defensive players\n",
    "defensive_players = load_players_data()\n",
    "print(f\"Found {len(defensive_players)} defensive players\")\n",
    "\n",
    "# Process each week's data\n",
    "all_metrics = []\n",
    "for week in range(1, 6):\n",
    "    print(f\"\\nProcessing week {week}\")\n",
    "    \n",
    "    # Load week data from parquet\n",
    "    week_data_path = os.path.join(DATA_DIR, f'nfc_west_week_{week}.parquet')\n",
    "    week_data = pd.read_parquet(week_data_path)\n",
    "    \n",
    "    # Calculate metrics\n",
    "    week_metrics = calculate_movement_metrics(week_data, defensive_players)\n",
    "    week_metrics['week'] = week\n",
    "    all_metrics.append(week_metrics)\n",
    "    \n",
    "    # Clear memory\n",
    "    del week_data\n",
    "    clear_memory()\n",
    "\n",
    "# Combine all weeks\n",
    "movement_df = pd.concat(all_metrics, ignore_index=True)\n",
    "\n",
    "# Create speed distribution plot (Plot 1)\n",
    "plt.figure(figsize=(12, 8))\n",
    "sns.boxplot(data=movement_df, x='team', y='avg_speed', hue='position')\n",
    "plt.title('Defensive Speed Distribution by Team and Position')\n",
    "plt.xlabel('Team')\n",
    "plt.ylabel('Average Speed (yards/second)')\n",
    "plt.xticks(rotation=45)\n",
    "plt.legend(title='Position', bbox_to_anchor=(1.05, 1), loc='upper left')\n",
    "plt.tight_layout()\n",
    "plt.savefig(os.path.join(os.path.dirname(os.getcwd()), 'plots', 'defensive_speed_distribution.png'))\n",
    "plt.close()\n",
    "\n",
    "# Save metrics to results\n",
    "metrics_path = os.path.join(os.path.dirname(os.getcwd()), 'results', 'movement_metrics.csv')\n",
    "movement_df.to_csv(metrics_path, index=False)\n",
    "\n",
    "# Print summary statistics\n",
    "print(\"\\nMovement Metrics Summary by Team:\")\n",
    "team_summary = movement_df.groupby('team').agg({\n",
    "    'avg_speed': ['mean', 'std'],\n",
    "    'max_speed': ['mean', 'std'],\n",
    "    'direction_changes': ['mean', 'std']\n",
    "}).round(2)\n",
    "print(team_summary)\n",
    "\n",
    "# Print memory usage\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 4: Zone Coverage Analysis (Optimized)\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.spatial import ConvexHull\n",
    "import seaborn as sns\n",
    "from tqdm import tqdm\n",
    "\n",
    "def create_coverage_heatmap(tracking_df, team, save_path):\n",
    "    \"\"\"Create defensive coverage heatmap for a team using sampled data\"\"\"\n",
    "    # Sample data points for faster processing (e.g., every 10th frame)\n",
    "    sampled_data = tracking_df[tracking_df['club'] == team].iloc[::10]\n",
    "    \n",
    "    plt.figure(figsize=(15, 8))\n",
    "    \n",
    "    # Plot football field boundaries\n",
    "    plt.plot([0, 0], [0, 53.3], 'white', alpha=0.5)\n",
    "    plt.plot([120, 120], [0, 53.3], 'white', alpha=0.5)\n",
    "    plt.plot([0, 120], [0, 0], 'white', alpha=0.5)\n",
    "    plt.plot([0, 120], [53.3, 53.3], 'white', alpha=0.5)\n",
    "    \n",
    "    # Create heatmap with fewer bins\n",
    "    plt.hist2d(sampled_data['x'], sampled_data['y'], \n",
    "               bins=40, range=[[0, 120], [0, 53.3]], \n",
    "               cmap='YlOrRd', alpha=0.7)\n",
    "    \n",
    "    plt.colorbar(label='Defensive Position Frequency')\n",
    "    plt.title(f'{team} Defensive Coverage Heatmap')\n",
    "    plt.xlabel('Field Position (yards)')\n",
    "    plt.ylabel('Field Position (yards)')\n",
    "    \n",
    "    ax = plt.gca()\n",
    "    ax.set_facecolor('forestgreen')\n",
    "    \n",
    "    plt.savefig(save_path, bbox_inches='tight', dpi=300)\n",
    "    plt.close()\n",
    "\n",
    "def calculate_coverage_metrics(tracking_df, team):\n",
    "    \"\"\"Calculate coverage metrics using sampled plays\"\"\"\n",
    "    team_data = tracking_df[tracking_df['club'] == team]\n",
    "    \n",
    "    # Sample plays for analysis (e.g., every 5th play)\n",
    "    play_ids = team_data['playId'].unique()[::5]\n",
    "    \n",
    "    metrics = {\n",
    "        'team': team,\n",
    "        'total_plays': len(play_ids),\n",
    "        'avg_defenders': team_data.groupby('playId')['nflId'].nunique().mean(),\n",
    "        'coverage_area': 0,\n",
    "        'avg_spacing': 0\n",
    "    }\n",
    "    \n",
    "    # Calculate metrics for sampled plays\n",
    "    coverage_areas = []\n",
    "    spacings = []\n",
    "    \n",
    "    for play_id in play_ids:\n",
    "        play_data = team_data[team_data['playId'] == play_id]\n",
    "        if len(play_data) > 0:\n",
    "            # Sample frames for this play\n",
    "            sampled_frames = play_data.iloc[::5]\n",
    "            points = sampled_frames[['x', 'y']].values\n",
    "            \n",
    "            if len(points) >= 3:\n",
    "                try:\n",
    "                    hull = ConvexHull(points)\n",
    "                    coverage_areas.append(hull.area)\n",
    "                    \n",
    "                    # Calculate spacing for subset of points\n",
    "                    if len(points) > 10:\n",
    "                        points = points[:10]\n",
    "                    spacing = np.mean([np.min([np.linalg.norm(p1 - p2) \n",
    "                                             for p2 in points if not np.array_equal(p1, p2)]) \n",
    "                                    for p1 in points])\n",
    "                    spacings.append(spacing)\n",
    "                except:\n",
    "                    continue\n",
    "    \n",
    "    metrics['coverage_area'] = np.mean(coverage_areas) if coverage_areas else 0\n",
    "    metrics['avg_spacing'] = np.mean(spacings) if spacings else 0\n",
    "    \n",
    "    return metrics\n",
    "\n",
    "# Process all weeks data first\n",
    "print(\"Loading tracking data...\")\n",
    "all_tracking_data = []\n",
    "for week in tqdm(range(1, 6), desc=\"Loading weeks\"):\n",
    "    week_data_path = os.path.join(DATA_DIR, f'nfc_west_week_{week}.parquet')\n",
    "    week_data = pd.read_parquet(week_data_path)\n",
    "    all_tracking_data.append(week_data)\n",
    "    clear_memory()\n",
    "\n",
    "# Combine all weeks\n",
    "tracking_df = pd.concat(all_tracking_data, ignore_index=True)\n",
    "del all_tracking_data\n",
    "clear_memory()\n",
    "\n",
    "# Process each team\n",
    "team_metrics = []\n",
    "for team in tqdm(NFC_WEST, desc=\"Processing teams\"):\n",
    "    print(f\"\\nAnalyzing {team}\")\n",
    "    \n",
    "    # Create heatmap\n",
    "    heatmap_path = os.path.join(os.path.dirname(os.getcwd()), 'plots', f'{team}_coverage_heatmap.png')\n",
    "    create_coverage_heatmap(tracking_df, team, heatmap_path)\n",
    "    \n",
    "    # Calculate metrics\n",
    "    metrics = calculate_coverage_metrics(tracking_df, team)\n",
    "    team_metrics.append(metrics)\n",
    "    clear_memory()\n",
    "\n",
    "# Create metrics DataFrame\n",
    "metrics_df = pd.DataFrame(team_metrics)\n",
    "\n",
    "# Create summary visualization\n",
    "plt.figure(figsize=(12, 6))\n",
    "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))\n",
    "\n",
    "# Coverage area comparison\n",
    "sns.barplot(data=metrics_df, x='team', y='coverage_area', ax=ax1)\n",
    "ax1.set_title('Average Coverage Area by Team')\n",
    "ax1.set_ylabel('Coverage Area (sq. yards)')\n",
    "\n",
    "# Defender spacing comparison\n",
    "sns.barplot(data=metrics_df, x='team', y='avg_spacing', ax=ax2)\n",
    "ax2.set_title('Average Defender Spacing by Team')\n",
    "ax2.set_ylabel('Spacing (yards)')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig(os.path.join(os.path.dirname(os.getcwd()), 'plots', 'coverage_metrics_comparison.png'))\n",
    "plt.close()\n",
    "\n",
    "# Save metrics to results\n",
    "metrics_df.to_csv(os.path.join(os.path.dirname(os.getcwd()), 'results', 'coverage_metrics.csv'), index=False)\n",
    "\n",
    "# Print summary and identify top teams\n",
    "print(\"\\nCoverage Metrics Summary:\")\n",
    "print(metrics_df.round(2))\n",
    "\n",
    "# Rank teams based on combined metrics\n",
    "metrics_df['overall_score'] = (\n",
    "    metrics_df['coverage_area'] / metrics_df['coverage_area'].max() +\n",
    "    metrics_df['avg_spacing'] / metrics_df['avg_spacing'].max()\n",
    ") / 2\n",
    "\n",
    "top_teams = metrics_df.nlargest(2, 'overall_score')\n",
    "print(\"\\nTop 2 performing teams:\")\n",
    "print(top_teams[['team', 'overall_score']].round(3))\n",
    "\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 5: Defender Tracking - Focus on Top Team\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Get top performing team from previous analysis\n",
    "top_team = metrics_df.nlargest(1, 'overall_score')['team'].iloc[0]\n",
    "print(f\"Focusing analysis on top team: {top_team}\")\n",
    "\n",
    "def find_notable_plays(tracking_df, plays_df, team):\n",
    "    \"\"\"Find notable defensive plays based on movement patterns\"\"\"\n",
    "    # Merge tracking with plays data\n",
    "    defensive_plays = plays_df[plays_df['defensiveTeam'] == team][['gameId', 'playId', 'passResult']]\n",
    "    play_data = pd.merge(tracking_df[tracking_df['club'] == team], \n",
    "                        defensive_plays, on=['gameId', 'playId'])\n",
    "    \n",
    "    play_metrics = []\n",
    "    \n",
    "    # Analyze each play\n",
    "    for (game_id, play_id), play_group in tqdm(play_data.groupby(['gameId', 'playId']), \n",
    "                                              desc=\"Analyzing plays\"):\n",
    "        # Calculate movement metrics\n",
    "        avg_speed = play_group['s'].mean()\n",
    "        max_speed = play_group['s'].max()\n",
    "        direction_changes = np.sum(np.abs(np.diff(play_group['dir'])) > 10)\n",
    "        \n",
    "        # Calculate defender spread\n",
    "        frames = play_group.groupby('frameId')\n",
    "        avg_spread = frames.apply(lambda x: np.std(x[['x', 'y']].values)).mean()\n",
    "        \n",
    "        play_metrics.append({\n",
    "            'gameId': game_id,\n",
    "            'playId': play_id,\n",
    "            'avg_speed': avg_speed,\n",
    "            'max_speed': max_speed,\n",
    "            'direction_changes': direction_changes,\n",
    "            'defender_spread': avg_spread,\n",
    "            'pass_result': play_group['passResult'].iloc[0],\n",
    "            'n_frames': len(frames)\n",
    "        })\n",
    "    \n",
    "    return pd.DataFrame(play_metrics)\n",
    "\n",
    "def plot_notable_play(tracking_df, play_info, save_path):\n",
    "    \"\"\"Create trajectory visualization for a notable play\"\"\"\n",
    "    play_data = tracking_df[(tracking_df['gameId'] == play_info['gameId']) & \n",
    "                           (tracking_df['playId'] == play_info['playId'])]\n",
    "    \n",
    "    plt.figure(figsize=(15, 8))\n",
    "    \n",
    "    # Plot field\n",
    "    plt.plot([0, 0], [0, 53.3], 'white', alpha=0.5)\n",
    "    plt.plot([120, 120], [0, 53.3], 'white', alpha=0.5)\n",
    "    plt.plot([0, 120], [0, 0], 'white', alpha=0.5)\n",
    "    plt.plot([0, 120], [53.3, 53.3], 'white', alpha=0.5)\n",
    "    \n",
    "    # Plot defender trajectories\n",
    "    for player_id, player_data in play_data.groupby('nflId'):\n",
    "        positions = player_data[['x', 'y']].values\n",
    "        plt.plot(positions[:, 0], positions[:, 1], '-', alpha=0.6, linewidth=2)\n",
    "        \n",
    "        # Mark start and end positions\n",
    "        plt.scatter(positions[0, 0], positions[0, 1], c='green', s=100, label='Start' if player_id == play_data['nflId'].iloc[0] else \"\")\n",
    "        plt.scatter(positions[-1, 0], positions[-1, 1], c='red', s=100, label='End' if player_id == play_data['nflId'].iloc[0] else \"\")\n",
    "    \n",
    "    plt.title(f\"Defensive Trajectories - {top_team}\\nPlay ID: {play_info['playId']} (Pass Result: {play_info['pass_result']})\")\n",
    "    plt.xlabel('Field Position (yards)')\n",
    "    plt.ylabel('Field Position (yards)')\n",
    "    \n",
    "    ax = plt.gca()\n",
    "    ax.set_facecolor('forestgreen')\n",
    "    plt.legend()\n",
    "    \n",
    "    plt.savefig(save_path, bbox_inches='tight', dpi=300)\n",
    "    plt.close()\n",
    "\n",
    "# Load necessary data\n",
    "print(\"Loading data...\")\n",
    "tracking_df = pd.read_parquet(os.path.join(DATA_DIR, 'nfc_west_week_1.parquet'))  # Start with week 1\n",
    "plays_df = pd.read_csv(os.path.join(DATA_DIR, 'plays.csv'))\n",
    "\n",
    "# Find notable plays\n",
    "play_metrics = find_notable_plays(tracking_df, plays_df, top_team)\n",
    "\n",
    "# Select top 3 plays based on different criteria\n",
    "interesting_plays = pd.concat([\n",
    "    play_metrics.nlargest(1, 'defender_spread'),  # Most spread out defense\n",
    "    play_metrics.nlargest(1, 'direction_changes'),  # Most complex movements\n",
    "    play_metrics[play_metrics['pass_result'].isin(['I', 'IN'])].nlargest(1, 'max_speed')  # Successful defense with high speed\n",
    "])\n",
    "\n",
    "# Create visualizations for selected plays\n",
    "print(\"\\nCreating play visualizations...\")\n",
    "for idx, play in interesting_plays.iterrows():\n",
    "    save_path = os.path.join(os.path.dirname(os.getcwd()), 'plots', \n",
    "                            f\"{top_team}_play_{play['playId']}_trajectories.png\")\n",
    "    plot_notable_play(tracking_df, play, save_path)\n",
    "\n",
    "# Save play metrics\n",
    "interesting_plays.to_csv(os.path.join(os.path.dirname(os.getcwd()), 'results', \n",
    "                                    f'{top_team}_notable_plays.csv'), index=False)\n",
    "\n",
    "print(\"\\nPlay Analysis Summary:\")\n",
    "print(interesting_plays[['playId', 'pass_result', 'defender_spread', \n",
    "                        'direction_changes', 'max_speed']].round(2))\n",
    "\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 6: Formation Analysis\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy.spatial import ConvexHull\n",
    "from tqdm import tqdm\n",
    "\n",
    "def analyze_formation(play_tracking, frame_id):\n",
    "    \"\"\"Analyze defensive formation for a specific frame\"\"\"\n",
    "    frame_data = play_tracking[play_tracking['frameId'] == frame_id]\n",
    "    \n",
    "    if len(frame_data) < 3:  # Need at least 3 defenders for meaningful formation\n",
    "        return None\n",
    "        \n",
    "    positions = frame_data[['x', 'y']].values\n",
    "    \n",
    "    # Calculate formation metrics\n",
    "    try:\n",
    "        hull = ConvexHull(positions)\n",
    "        formation_metrics = {\n",
    "            'area': hull.area,  # Area covered by formation\n",
    "            'width': np.ptp(positions[:, 0]),  # Formation width\n",
    "            'depth': np.ptp(positions[:, 1]),  # Formation depth\n",
    "            'avg_spacing': np.mean([np.min([np.linalg.norm(p1 - p2) \n",
    "                                          for j, p2 in enumerate(positions) if i != j])\n",
    "                                  for i, p1 in enumerate(positions)]),\n",
    "            'n_defenders': len(positions)\n",
    "        }\n",
    "        return formation_metrics\n",
    "    except:\n",
    "        return None\n",
    "\n",
    "def plot_formation_analysis(play_tracking, play_info, save_path):\n",
    "    \"\"\"Create formation analysis visualization\"\"\"\n",
    "    # Get pre-snap formation\n",
    "    pre_snap = play_tracking[play_tracking['frameType'] == 'BEFORE_SNAP']\n",
    "    if len(pre_snap) == 0:\n",
    "        return\n",
    "        \n",
    "    # Get last frame before snap\n",
    "    last_pre_snap = pre_snap.groupby('frameId').last().reset_index()\n",
    "    \n",
    "    plt.figure(figsize=(15, 8))\n",
    "    \n",
    "    # Plot field\n",
    "    plt.plot([0, 0], [0, 53.3], 'white', alpha=0.5)\n",
    "    plt.plot([120, 120], [0, 53.3], 'white', alpha=0.5)\n",
    "    plt.plot([0, 120], [0, 0], 'white', alpha=0.5)\n",
    "    plt.plot([0, 120], [53.3, 53.3], 'white', alpha=0.5)\n",
    "    \n",
    "    # Plot defender positions\n",
    "    positions = last_pre_snap[['x', 'y']].values\n",
    "    plt.scatter(positions[:, 0], positions[:, 1], c='white', s=100, label='Defenders')\n",
    "    \n",
    "    # Plot formation hull\n",
    "    if len(positions) >= 3:\n",
    "        hull = ConvexHull(positions)\n",
    "        for simplex in hull.simplices:\n",
    "            plt.plot(positions[simplex, 0], positions[simplex, 1], 'w--', alpha=0.5)\n",
    "    \n",
    "    # Add formation metrics to plot\n",
    "    metrics = analyze_formation(play_tracking, last_pre_snap['frameId'].iloc[0])\n",
    "    if metrics:\n",
    "        plt.title(f\"Defensive Formation Analysis - {top_team}\\n\" +\n",
    "                 f\"Width: {metrics['width']:.1f} yards, Depth: {metrics['depth']:.1f} yards\\n\" +\n",
    "                 f\"Area: {metrics['area']:.1f} sq. yards, Avg Spacing: {metrics['avg_spacing']:.1f} yards\")\n",
    "    \n",
    "    plt.xlabel('Field Position (yards)')\n",
    "    plt.ylabel('Field Position (yards)')\n",
    "    \n",
    "    ax = plt.gca()\n",
    "    ax.set_facecolor('forestgreen')\n",
    "    \n",
    "    plt.savefig(save_path, bbox_inches='tight', dpi=300)\n",
    "    plt.close()\n",
    "\n",
    "# Load data for the top team's plays\n",
    "print(f\"Analyzing formations for {top_team}...\")\n",
    "\n",
    "# Use the notable plays identified in Cell 5\n",
    "notable_plays_df = pd.read_csv(os.path.join(os.path.dirname(os.getcwd()), \n",
    "                                           'results', f'{top_team}_notable_plays.csv'))\n",
    "\n",
    "# Analyze formations for each notable play\n",
    "formation_metrics = []\n",
    "for _, play in tqdm(notable_plays_df.iterrows(), desc=\"Analyzing formations\"):\n",
    "    # Load play tracking data\n",
    "    play_tracking = tracking_df[\n",
    "        (tracking_df['gameId'] == play['gameId']) & \n",
    "        (tracking_df['playId'] == play['playId']) &\n",
    "        (tracking_df['club'] == top_team)\n",
    "    ]\n",
    "    \n",
    "    # Create formation visualization\n",
    "    save_path = os.path.join(os.path.dirname(os.getcwd()), 'plots', \n",
    "                            f\"{top_team}_play_{play['playId']}_formation.png\")\n",
    "    plot_formation_analysis(play_tracking, play, save_path)\n",
    "    \n",
    "    # Analyze pre-snap formation\n",
    "    pre_snap_frames = play_tracking[play_tracking['frameType'] == 'BEFORE_SNAP']['frameId'].unique()\n",
    "    if len(pre_snap_frames) > 0:\n",
    "        metrics = analyze_formation(play_tracking, pre_snap_frames[-1])  # Last pre-snap frame\n",
    "        if metrics:\n",
    "            metrics['gameId'] = play['gameId']\n",
    "            metrics['playId'] = play['playId']\n",
    "            metrics['pass_result'] = play['pass_result']\n",
    "            formation_metrics.append(metrics)\n",
    "\n",
    "# Create formation metrics DataFrame\n",
    "formation_df = pd.DataFrame(formation_metrics)\n",
    "\n",
    "# Create summary visualization\n",
    "plt.figure(figsize=(12, 8))\n",
    "fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 12))\n",
    "\n",
    "# Formation area vs success\n",
    "sns.boxplot(data=formation_df, x='pass_result', y='area', ax=ax1)\n",
    "ax1.set_title('Formation Area by Play Result')\n",
    "ax1.set_xlabel('Pass Result')\n",
    "ax1.set_ylabel('Area (sq. yards)')\n",
    "\n",
    "# Formation spacing distribution\n",
    "sns.histplot(data=formation_df, x='avg_spacing', bins=20, ax=ax2)\n",
    "ax2.set_title('Distribution of Defender Spacing')\n",
    "ax2.set_xlabel('Average Spacing (yards)')\n",
    "\n",
    "# Width vs Depth scatter\n",
    "sns.scatterplot(data=formation_df, x='width', y='depth', \n",
    "                size='area', sizes=(100, 1000), alpha=0.6, ax=ax3)\n",
    "ax3.set_title('Formation Dimensions')\n",
    "ax3.set_xlabel('Width (yards)')\n",
    "ax3.set_ylabel('Depth (yards)')\n",
    "\n",
    "# Number of defenders\n",
    "sns.countplot(data=formation_df, x='n_defenders', ax=ax4)\n",
    "ax4.set_title('Number of Defenders in Formation')\n",
    "ax4.set_xlabel('Number of Defenders')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig(os.path.join(os.path.dirname(os.getcwd()), 'plots', \n",
    "                        f'{top_team}_formation_analysis.png'))\n",
    "plt.close()\n",
    "\n",
    "# Save formation metrics\n",
    "formation_df.to_csv(os.path.join(os.path.dirname(os.getcwd()), 'results', \n",
    "                                f'{top_team}_formation_metrics.csv'), index=False)\n",
    "\n",
    "print(\"\\nFormation Analysis Summary:\")\n",
    "print(formation_df.describe().round(2))\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 7: Environmental Constraints Analysis\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.spatial import Voronoi\n",
    "import matplotlib.patches as patches\n",
    "from tqdm import tqdm\n",
    "\n",
    "def calculate_safe_zones(positions, field_dims=(120, 53.3)):\n",
    "    \"\"\"Calculate areas where defenders can move safely\"\"\"\n",
    "    try:\n",
    "        # Create Voronoi diagram for defensive positions\n",
    "        vor = Voronoi(positions)\n",
    "        \n",
    "        # Initialize bounds\n",
    "        bounds = [0, field_dims[0], 0, field_dims[1]]\n",
    "        \n",
    "        # Filter vertices within field bounds\n",
    "        valid_vertices = []\n",
    "        for vertex in vor.vertices:\n",
    "            if (bounds[0] <= vertex[0] <= bounds[1] and \n",
    "                bounds[2] <= vertex[3] <= bounds[3]):\n",
    "                valid_vertices.append(vertex)\n",
    "                \n",
    "        return np.array(valid_vertices) if valid_vertices else None\n",
    "    except:\n",
    "        return None\n",
    "\n",
    "def analyze_constraints(play_tracking, field_dims=(120, 53.3)):\n",
    "    \"\"\"Analyze environmental constraints on defender movement\"\"\"\n",
    "    constraints = {\n",
    "        'sideline_proximity': [],  # Distance to nearest sideline\n",
    "        'defender_spacing': [],    # Distance to nearest defender\n",
    "        'movement_angles': [],     # Available movement angles\n",
    "        'field_position': []      # Position relative to field length\n",
    "    }\n",
    "    \n",
    "    for _, frame in play_tracking.groupby('frameId'):\n",
    "        positions = frame[['x', 'y']].values\n",
    "        \n",
    "        for pos in positions:\n",
    "            # Sideline proximity\n",
    "            sideline_dist = min(pos[1], field_dims[1] - pos[1])\n",
    "            constraints['sideline_proximity'].append(sideline_dist)\n",
    "            \n",
    "            # Defender spacing\n",
    "            other_positions = positions[~np.all(positions == pos, axis=1)]\n",
    "            if len(other_positions) > 0:\n",
    "                min_spacing = np.min(np.linalg.norm(other_positions - pos, axis=1))\n",
    "                constraints['defender_spacing'].append(min_spacing)\n",
    "            \n",
    "            # Field position\n",
    "            field_pos = pos[0] / field_dims[0]\n",
    "            constraints['field_position'].append(field_pos)\n",
    "            \n",
    "            # Movement angles (simplified)\n",
    "            available_angles = []\n",
    "            for angle in range(0, 360, 45):\n",
    "                rad = np.radians(angle)\n",
    "                direction = np.array([np.cos(rad), np.sin(rad)])\n",
    "                available_angles.append(angle)\n",
    "            constraints['movement_angles'].append(len(available_angles))\n",
    "    \n",
    "    return constraints\n",
    "\n",
    "def plot_constraints(play_tracking, constraints, save_path):\n",
    "    \"\"\"Visualize defensive movement constraints\"\"\"\n",
    "    plt.figure(figsize=(15, 10))\n",
    "    \n",
    "    # Create subplot grid\n",
    "    fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 12))\n",
    "    \n",
    "    # 1. Sideline Proximity Distribution\n",
    "    sns.histplot(constraints['sideline_proximity'], bins=20, ax=ax1)\n",
    "    ax1.set_title('Distance to Sideline Distribution')\n",
    "    ax1.set_xlabel('Distance to Sideline (yards)')\n",
    "    \n",
    "    # 2. Defender Spacing Distribution\n",
    "    sns.histplot(constraints['defender_spacing'], bins=20, ax=ax2)\n",
    "    ax2.set_title('Inter-Defender Spacing Distribution')\n",
    "    ax2.set_xlabel('Distance to Nearest Defender (yards)')\n",
    "    \n",
    "    # 3. Field Position Heatmap\n",
    "    positions = play_tracking[['x', 'y']].values\n",
    "    ax3.hist2d(positions[:, 0], positions[:, 1], bins=30, cmap='YlOrRd')\n",
    "    ax3.set_title('Defensive Position Heatmap')\n",
    "    ax3.set_xlabel('Field Position (yards)')\n",
    "    ax3.set_ylabel('Field Width (yards)')\n",
    "    \n",
    "    # 4. Movement Constraints Visualization\n",
    "    sample_frame = play_tracking.groupby('frameId').first().iloc[0]\n",
    "    frame_data = play_tracking[play_tracking['frameId'] == sample_frame.name]\n",
    "    positions = frame_data[['x', 'y']].values\n",
    "    \n",
    "    ax4.scatter(positions[:, 0], positions[:, 1], c='white', s=100)\n",
    "    \n",
    "    # Add field boundaries\n",
    "    for ax in [ax3, ax4]:\n",
    "        ax.plot([0, 0], [0, 53.3], 'white', alpha=0.5)\n",
    "        ax.plot([120, 120], [0, 53.3], 'white', alpha=0.5)\n",
    "        ax.plot([0, 120], [0, 0], 'white', alpha=0.5)\n",
    "        ax.plot([0, 120], [53.3, 53.3], 'white', alpha=0.5)\n",
    "        ax.set_facecolor('forestgreen')\n",
    "    \n",
    "    ax4.set_title('Sample Frame Movement Constraints')\n",
    "    ax4.set_xlabel('Field Position (yards)')\n",
    "    ax4.set_ylabel('Field Width (yards)')\n",
    "    \n",
    "    plt.tight_layout()\n",
    "    plt.savefig(save_path, bbox_inches='tight', dpi=300)\n",
    "    plt.close()\n",
    "\n",
    "# Load data for analysis\n",
    "print(f\"Analyzing environmental constraints for {top_team}...\")\n",
    "\n",
    "# Use notable plays from previous analysis\n",
    "notable_plays_df = pd.read_csv(os.path.join(os.path.dirname(os.getcwd()), \n",
    "                                           'results', f'{top_team}_notable_plays.csv'))\n",
    "\n",
    "# Analyze constraints for each play\n",
    "all_constraints = []\n",
    "for _, play in tqdm(notable_plays_df.iterrows(), desc=\"Analyzing plays\"):\n",
    "    play_tracking = tracking_df[\n",
    "        (tracking_df['gameId'] == play['gameId']) & \n",
    "        (tracking_df['playId'] == play['playId']) &\n",
    "        (tracking_df['club'] == top_team)\n",
    "    ]\n",
    "    \n",
    "    # Calculate constraints\n",
    "    constraints = analyze_constraints(play_tracking)\n",
    "    \n",
    "    # Create visualization\n",
    "    save_path = os.path.join(os.path.dirname(os.getcwd()), 'plots',\n",
    "                            f\"{top_team}_play_{play['playId']}_constraints.png\")\n",
    "    plot_constraints(play_tracking, constraints, save_path)\n",
    "    \n",
    "    # Aggregate constraints\n",
    "    constraints_summary = {\n",
    "        'gameId': play['gameId'],\n",
    "        'playId': play['playId'],\n",
    "        'avg_sideline_dist': np.mean(constraints['sideline_proximity']),\n",
    "        'min_defender_spacing': np.min(constraints['defender_spacing']),\n",
    "        'avg_movement_angles': np.mean(constraints['movement_angles'])\n",
    "    }\n",
    "    all_constraints.append(constraints_summary)\n",
    "\n",
    "# Create constraints summary DataFrame\n",
    "constraints_df = pd.DataFrame(all_constraints)\n",
    "\n",
    "# Save results\n",
    "constraints_df.to_csv(os.path.join(os.path.dirname(os.getcwd()), 'results',\n",
    "                                  f'{top_team}_constraints_analysis.csv'), index=False)\n",
    "\n",
    "print(\"\\nConstraints Analysis Summary:\")\n",
    "print(constraints_df.describe().round(2))\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 8: Movement Pattern Classification\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.cluster import KMeans\n",
    "from scipy.signal import savgol_filter\n",
    "import seaborn as sns\n",
    "from tqdm import tqdm\n",
    "\n",
    "def extract_movement_features(play_tracking):\n",
    "    \"\"\"Extract key features from defender movements\"\"\"\n",
    "    features = []\n",
    "    \n",
    "    # Process each defender's movement\n",
    "    for player_id, player_data in play_tracking.groupby('nflId'):\n",
    "        # Sort by frame\n",
    "        player_data = player_data.sort_values('frameId')\n",
    "        \n",
    "        # Calculate movement features\n",
    "        speed_profile = player_data['s'].values\n",
    "        direction_profile = player_data['dir'].values\n",
    "        acceleration_profile = player_data['a'].values\n",
    "        \n",
    "        # Smooth profiles using Savitzky-Golay filter\n",
    "        if len(speed_profile) > 5:  # Need minimum length for filtering\n",
    "            speed_smooth = savgol_filter(speed_profile, 5, 2)\n",
    "            acc_smooth = savgol_filter(acceleration_profile, 5, 2)\n",
    "        else:\n",
    "            speed_smooth = speed_profile\n",
    "            acc_smooth = acceleration_profile\n",
    "            \n",
    "        # Extract features\n",
    "        features.append({\n",
    "            'nflId': player_id,\n",
    "            'avg_speed': np.mean(speed_smooth),\n",
    "            'max_speed': np.max(speed_smooth),\n",
    "            'speed_var': np.var(speed_smooth),\n",
    "            'avg_acc': np.mean(acc_smooth),\n",
    "            'max_acc': np.max(acc_smooth),\n",
    "            'direction_changes': np.sum(np.abs(np.diff(direction_profile)) > 10),\n",
    "            'total_distance': np.sum(player_data['dis'])\n",
    "        })\n",
    "    \n",
    "    return pd.DataFrame(features)\n",
    "\n",
    "def classify_movements(features, n_clusters=4):\n",
    "    \"\"\"Classify movement patterns using KMeans\"\"\"\n",
    "    # Standardize features\n",
    "    scaler = StandardScaler()\n",
    "    X = scaler.fit_transform(features.drop('nflId', axis=1))\n",
    "    \n",
    "    # Apply KMeans clustering\n",
    "    kmeans = KMeans(n_clusters=n_clusters, random_state=42)\n",
    "    labels = kmeans.fit_predict(X)\n",
    "    \n",
    "    # Add cluster labels to features\n",
    "    features['pattern'] = labels\n",
    "    \n",
    "    return features, kmeans.cluster_centers_\n",
    "\n",
    "def plot_pattern_analysis(features, centers, save_path):\n",
    "    \"\"\"Create visualization of movement patterns\"\"\"\n",
    "    fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 12))\n",
    "    \n",
    "    # 1. Speed vs Acceleration by Pattern\n",
    "    scatter1 = ax1.scatter(features['avg_speed'], features['avg_acc'], \n",
    "                          c=features['pattern'], cmap='viridis')\n",
    "    ax1.set_xlabel('Average Speed (yards/s)')\n",
    "    ax1.set_ylabel('Average Acceleration (yards/s²)')\n",
    "    ax1.set_title('Speed vs Acceleration Patterns')\n",
    "    plt.colorbar(scatter1, ax=ax1, label='Pattern')\n",
    "\n",
    "    # 2. Direction Changes Distribution\n",
    "    sns.boxplot(data=features, x='pattern', y='direction_changes', ax=ax2)\n",
    "    ax2.set_title('Direction Changes by Pattern')\n",
    "    ax2.set_xlabel('Pattern')\n",
    "    ax2.set_ylabel('Number of Direction Changes')\n",
    "\n",
    "    # 3. Pattern Characteristics Heatmap\n",
    "    pattern_means = features.groupby('pattern').mean()\n",
    "    sns.heatmap(pattern_means.drop('nflId', axis=1), \n",
    "                annot=True, fmt='.2f', cmap='YlOrRd', ax=ax3)\n",
    "    ax3.set_title('Pattern Characteristics')\n",
    "\n",
    "    # 4. Distance vs Speed Variability\n",
    "    scatter2 = ax4.scatter(features['total_distance'], features['speed_var'],\n",
    "                          c=features['pattern'], cmap='viridis', alpha=0.6)\n",
    "    ax4.set_xlabel('Total Distance (yards)')\n",
    "    ax4.set_ylabel('Speed Variability')\n",
    "    ax4.set_title('Distance vs Speed Variability')\n",
    "    plt.colorbar(scatter2, ax=ax4, label='Pattern')\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(save_path, bbox_inches='tight', dpi=300)\n",
    "    plt.close()\n",
    "\n",
    "# Load data and process patterns\n",
    "print(f\"Analyzing movement patterns for {top_team}...\")\n",
    "\n",
    "# Use notable plays from previous analysis\n",
    "notable_plays_df = pd.read_csv(os.path.join(os.path.dirname(os.getcwd()),\n",
    "                                           'results', f'{top_team}_notable_plays.csv'))\n",
    "\n",
    "# Analyze patterns for each play\n",
    "all_patterns = []\n",
    "for _, play in tqdm(notable_plays_df.iterrows(), desc=\"Analyzing patterns\"):\n",
    "    play_tracking = tracking_df[\n",
    "        (tracking_df['gameId'] == play['gameId']) & \n",
    "        (tracking_df['playId'] == play['playId']) &\n",
    "        (tracking_df['club'] == top_team)\n",
    "    ]\n",
    "    \n",
    "    # Extract features and classify movements\n",
    "    features = extract_movement_features(play_tracking)\n",
    "    features['gameId'] = play['gameId']\n",
    "    features['playId'] = play['playId']\n",
    "    \n",
    "    classified_features, centers = classify_movements(features.drop(['gameId', 'playId'], axis=1))\n",
    "    classified_features['gameId'] = play['gameId']\n",
    "    classified_features['playId'] = play['playId']\n",
    "    \n",
    "    all_patterns.append(classified_features)\n",
    "    \n",
    "    # Create visualization\n",
    "    save_path = os.path.join(os.path.dirname(os.getcwd()), 'plots',\n",
    "                            f\"{top_team}_play_{play['playId']}_patterns.png\")\n",
    "    plot_pattern_analysis(classified_features, centers, save_path)\n",
    "\n",
    "# Combine all patterns\n",
    "patterns_df = pd.concat(all_patterns, ignore_index=True)\n",
    "\n",
    "# Calculate pattern statistics\n",
    "pattern_stats = patterns_df.groupby('pattern').agg({\n",
    "    'avg_speed': ['mean', 'std'],\n",
    "    'max_speed': ['mean', 'std'],\n",
    "    'direction_changes': ['mean', 'std'],\n",
    "    'total_distance': ['mean', 'std']\n",
    "}).round(2)\n",
    "\n",
    "# Save results\n",
    "patterns_df.to_csv(os.path.join(os.path.dirname(os.getcwd()), 'results',\n",
    "                               f'{top_team}_movement_patterns.csv'), index=False)\n",
    "pattern_stats.to_csv(os.path.join(os.path.dirname(os.getcwd()), 'results',\n",
    "                                 f'{top_team}_pattern_statistics.csv'))\n",
    "\n",
    "print(\"\\nPattern Analysis Summary:\")\n",
    "print(\"\\nPattern Distribution:\")\n",
    "print(patterns_df['pattern'].value_counts().sort_index())\n",
    "print(\"\\nPattern Statistics:\")\n",
    "print(pattern_stats)\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 9: Efficiency Metrics and Optimal Path Analysis\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.spatial.distance import cdist\n",
    "from scipy.interpolate import interp1d\n",
    "import seaborn as sns\n",
    "from tqdm import tqdm\n",
    "\n",
    "def calculate_optimal_path(start_pos, end_pos, obstacles, n_points=50):\n",
    "    \"\"\"Calculate theoretically optimal path avoiding obstacles\"\"\"\n",
    "    # Create smooth path between start and end\n",
    "    t = np.linspace(0, 1, n_points)\n",
    "    path_x = np.linspace(start_pos[0], end_pos[0], n_points)\n",
    "    path_y = np.linspace(start_pos[1], end_pos[1], n_points)\n",
    "    \n",
    "    # Adjust path to avoid obstacles\n",
    "    for i in range(1, n_points-1):\n",
    "        point = np.array([path_x[i], path_y[i]])\n",
    "        \n",
    "        # Check distance to obstacles\n",
    "        distances = cdist([point], obstacles)\n",
    "        min_dist = np.min(distances)\n",
    "        \n",
    "        if min_dist < 2:  # If too close to obstacle\n",
    "            # Find nearest obstacle\n",
    "            nearest_idx = np.argmin(distances)\n",
    "            obstacle = obstacles[nearest_idx]\n",
    "            \n",
    "            # Calculate avoidance vector\n",
    "            avoid_vector = point - obstacle\n",
    "            avoid_vector = avoid_vector / np.linalg.norm(avoid_vector)\n",
    "            \n",
    "            # Adjust point position\n",
    "            path_x[i] = point[0] + avoid_vector[0] * (2 - min_dist)\n",
    "            path_y[i] = point[1] + avoid_vector[1] * (2 - min_dist)\n",
    "    \n",
    "    return np.column_stack([path_x, path_y])\n",
    "\n",
    "def calculate_efficiency_metrics(actual_path, optimal_path):\n",
    "    \"\"\"Calculate movement efficiency metrics\"\"\"\n",
    "    # Calculate path lengths\n",
    "    actual_length = np.sum(np.sqrt(np.sum(np.diff(actual_path, axis=0)**2, axis=1)))\n",
    "    optimal_length = np.sum(np.sqrt(np.sum(np.diff(optimal_path, axis=0)**2, axis=1)))\n",
    "    \n",
    "    # Calculate average deviation from optimal path\n",
    "    deviations = []\n",
    "    for actual_point in actual_path:\n",
    "        distances = cdist([actual_point], optimal_path)\n",
    "        deviations.append(np.min(distances))\n",
    "    \n",
    "    metrics = {\n",
    "        'path_efficiency': optimal_length / actual_length if actual_length > 0 else 0,\n",
    "        'avg_deviation': np.mean(deviations),\n",
    "        'max_deviation': np.max(deviations),\n",
    "        'total_distance': actual_length\n",
    "    }\n",
    "    \n",
    "    return metrics\n",
    "\n",
    "def plot_efficiency_analysis(actual_path, optimal_path, metrics, save_path):\n",
    "    \"\"\"Create visualization comparing actual and optimal paths\"\"\"\n",
    "    plt.figure(figsize=(15, 10))\n",
    "    \n",
    "    # Create subplot grid\n",
    "    fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 12))\n",
    "    \n",
    "    # 1. Path Comparison\n",
    "    ax1.plot(actual_path[:, 0], actual_path[:, 1], 'b-', label='Actual Path', alpha=0.7)\n",
    "    ax1.plot(optimal_path[:, 0], optimal_path[:, 1], 'r--', label='Optimal Path', alpha=0.7)\n",
    "    ax1.scatter(actual_path[0, 0], actual_path[0, 1], c='g', s=100, label='Start')\n",
    "    ax1.scatter(actual_path[-1, 0], actual_path[-1, 1], c='r', s=100, label='End')\n",
    "    \n",
    "    # Add field boundaries\n",
    "    ax1.plot([0, 0], [0, 53.3], 'white', alpha=0.5)\n",
    "    ax1.plot([120, 120], [0, 53.3], 'white', alpha=0.5)\n",
    "    ax1.plot([0, 120], [0, 0], 'white', alpha=0.5)\n",
    "    ax1.plot([0, 120], [53.3, 53.3], 'white', alpha=0.5)\n",
    "    ax1.set_facecolor('forestgreen')\n",
    "    \n",
    "    ax1.set_title('Path Comparison')\n",
    "    ax1.set_xlabel('Field Position (yards)')\n",
    "    ax1.set_ylabel('Field Position (yards)')\n",
    "    ax1.legend()\n",
    "    \n",
    "    # 2. Deviation Over Time\n",
    "    deviations = [np.min(cdist([p], optimal_path)) for p in actual_path]\n",
    "    ax2.plot(deviations, 'b-')\n",
    "    ax2.set_title('Path Deviation Over Time')\n",
    "    ax2.set_xlabel('Time Step')\n",
    "    ax2.set_ylabel('Deviation (yards)')\n",
    "    \n",
    "    # 3. Efficiency Metrics\n",
    "    metrics_list = list(metrics.items())  # Convert dict_items to list\n",
    "    ax3.axis('off')\n",
    "    ax3.table(cellText=[[v] for k, v in metrics_list if isinstance(v, (int, float))],\n",
    "              rowLabels=[k for k, v in metrics_list if isinstance(v, (int, float))],\n",
    "              colLabels=['Value'],\n",
    "              loc='center')\n",
    "    ax3.set_title('Efficiency Metrics')\n",
    "    \n",
    "    # 4. Cumulative Distance\n",
    "    actual_cum_dist = np.cumsum(np.sqrt(np.sum(np.diff(actual_path, axis=0)**2, axis=1)))\n",
    "    optimal_cum_dist = np.cumsum(np.sqrt(np.sum(np.diff(optimal_path, axis=0)**2, axis=1)))\n",
    "    \n",
    "    ax4.plot(actual_cum_dist, 'b-', label='Actual')\n",
    "    ax4.plot(optimal_cum_dist, 'r--', label='Optimal')\n",
    "    ax4.set_title('Cumulative Distance')\n",
    "    ax4.set_xlabel('Time Step')\n",
    "    ax4.set_ylabel('Distance (yards)')\n",
    "    ax4.legend()\n",
    "    \n",
    "    plt.tight_layout()\n",
    "    plt.savefig(save_path, bbox_inches='tight', dpi=300)\n",
    "    plt.close()\n",
    "\n",
    "# Load data for analysis\n",
    "print(f\"Analyzing movement efficiency for {top_team}...\")\n",
    "\n",
    "# Use notable plays from previous analysis\n",
    "notable_plays_df = pd.read_csv(os.path.join(os.path.dirname(os.getcwd()),\n",
    "                                           'results', f'{top_team}_notable_plays.csv'))\n",
    "\n",
    "# Analyze efficiency for each play\n",
    "all_efficiency_metrics = []\n",
    "for _, play in tqdm(notable_plays_df.iterrows(), desc=\"Analyzing efficiency\"):\n",
    "    play_tracking = tracking_df[\n",
    "        (tracking_df['gameId'] == play['gameId']) & \n",
    "        (tracking_df['playId'] == play['playId']) &\n",
    "        (tracking_df['club'] == top_team)\n",
    "    ]\n",
    "    \n",
    "    # Process each defender\n",
    "    for player_id, player_data in play_tracking.groupby('nflId'):\n",
    "        # Get actual path\n",
    "        actual_path = player_data[['x', 'y']].values\n",
    "        \n",
    "        # Get other players' positions as obstacles\n",
    "        other_players = play_tracking[play_tracking['nflId'] != player_id]\n",
    "        obstacles = other_players[['x', 'y']].values\n",
    "        \n",
    "        # Calculate optimal path\n",
    "        optimal_path = calculate_optimal_path(actual_path[0], actual_path[-1], obstacles)\n",
    "        \n",
    "        # Calculate metrics\n",
    "        metrics = calculate_efficiency_metrics(actual_path, optimal_path)\n",
    "        metrics.update({\n",
    "            'gameId': play['gameId'],\n",
    "            'playId': play['playId'],\n",
    "            'nflId': player_id\n",
    "        })\n",
    "        all_efficiency_metrics.append(metrics)\n",
    "        \n",
    "        # Create visualization\n",
    "        save_path = os.path.join(os.path.dirname(os.getcwd()), 'plots',\n",
    "                                f\"{top_team}_play_{play['playId']}_player_{player_id}_efficiency.png\")\n",
    "        plot_efficiency_analysis(actual_path, optimal_path, metrics, save_path)\n",
    "\n",
    "# Create efficiency DataFrame\n",
    "efficiency_df = pd.DataFrame(all_efficiency_metrics)\n",
    "\n",
    "# Calculate summary statistics\n",
    "efficiency_summary = efficiency_df.groupby(['gameId', 'playId']).agg({\n",
    "    'path_efficiency': ['mean', 'std'],\n",
    "    'avg_deviation': ['mean', 'std'],\n",
    "    'total_distance': ['mean', 'sum']\n",
    "}).round(3)\n",
    "\n",
    "# Save results\n",
    "efficiency_df.to_csv(os.path.join(os.path.dirname(os.getcwd()), 'results',\n",
    "                                 f'{top_team}_efficiency_metrics.csv'), index=False)\n",
    "efficiency_summary.to_csv(os.path.join(os.path.dirname(os.getcwd()), 'results',\n",
    "                                     f'{top_team}_efficiency_summary.csv'))\n",
    "\n",
    "print(\"\\nEfficiency Analysis Summary:\")\n",
    "print(\"\\nOverall Metrics:\")\n",
    "print(efficiency_df[['path_efficiency', 'avg_deviation', 'total_distance']].describe().round(3))\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 10: Animation Generation\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.animation as animation\n",
    "from matplotlib.patches import Circle, Rectangle\n",
    "from tqdm import tqdm\n",
    "\n",
    "def create_field():\n",
    "    \"\"\"Create football field for animation\"\"\"\n",
    "    fig, ax = plt.subplots(figsize=(15, 8))\n",
    "    \n",
    "    # Field boundaries\n",
    "    ax.plot([0, 0], [0, 53.3], 'white', alpha=0.5)\n",
    "    ax.plot([120, 120], [0, 53.3], 'white', alpha=0.5)\n",
    "    ax.plot([0, 120], [0, 0], 'white', alpha=0.5)\n",
    "    ax.plot([0, 120], [53.3, 53.3], 'white', alpha=0.5)\n",
    "    \n",
    "    ax.set_facecolor('forestgreen')\n",
    "    ax.set_xlim(-5, 125)\n",
    "    ax.set_ylim(-5, 58.3)\n",
    "    \n",
    "    return fig, ax\n",
    "\n",
    "def create_defender_animation(play_tracking, save_path):\n",
    "    \"\"\"Create animation of defensive movement\"\"\"\n",
    "    fig, ax = create_field()\n",
    "    \n",
    "    # Prepare defender data\n",
    "    defenders = {}\n",
    "    trails = {}\n",
    "    for player_id in play_tracking['nflId'].unique():\n",
    "        player_data = play_tracking[play_tracking['nflId'] == player_id]\n",
    "        defenders[player_id] = player_data[['frameId', 'x', 'y']].values\n",
    "        trails[player_id] = []\n",
    "    \n",
    "    # Initialize plots\n",
    "    defender_plots = {}\n",
    "    trail_plots = {}\n",
    "    for player_id in defenders:\n",
    "        defender_plots[player_id], = ax.plot([], [], 'wo', markersize=10)\n",
    "        trail_plots[player_id], = ax.plot([], [], 'w-', alpha=0.3)\n",
    "    \n",
    "    # Add time display\n",
    "    time_text = ax.text(5, 55, '', color='white', fontsize=12)\n",
    "    \n",
    "    def init():\n",
    "        for player_id in defenders:\n",
    "            defender_plots[player_id].set_data([], [])\n",
    "            trail_plots[player_id].set_data([], [])\n",
    "        time_text.set_text('')\n",
    "        return list(defender_plots.values()) + list(trail_plots.values()) + [time_text]\n",
    "    \n",
    "    def animate(frame):\n",
    "        for player_id in defenders:\n",
    "            if frame < len(defenders[player_id]):\n",
    "                x, y = defenders[player_id][frame][1:3]\n",
    "                defender_plots[player_id].set_data([x], [y])\n",
    "                \n",
    "                # Update trails\n",
    "                trails[player_id].append([x, y])\n",
    "                if len(trails[player_id]) > 10:  # Trail length\n",
    "                    trails[player_id].pop(0)\n",
    "                trail_data = np.array(trails[player_id])\n",
    "                trail_plots[player_id].set_data(trail_data[:, 0], trail_data[:, 1])\n",
    "        \n",
    "        time_text.set_text(f'Frame: {frame}')\n",
    "        return list(defender_plots.values()) + list(trail_plots.values()) + [time_text]\n",
    "    \n",
    "    # Create animation\n",
    "    anim = animation.FuncAnimation(fig, animate, init_func=init,\n",
    "                                 frames=len(defenders[list(defenders.keys())[0]]),\n",
    "                                 interval=50, blit=True)\n",
    "    \n",
    "    # Save animation\n",
    "    writer = animation.PillowWriter(fps=20)\n",
    "    anim.save(save_path, writer=writer)\n",
    "    plt.close()\n",
    "\n",
    "def create_zone_coverage_animation(play_tracking, save_path):\n",
    "    \"\"\"Create animation of defensive zone coverage\"\"\"\n",
    "    fig, ax = create_field()\n",
    "    \n",
    "    frames = play_tracking['frameId'].unique()\n",
    "    x = np.linspace(0, 120, 50)\n",
    "    y = np.linspace(0, 53.3, 25)\n",
    "    X, Y = np.meshgrid(x, y)\n",
    "    \n",
    "    # Initialize heatmap\n",
    "    heatmap = ax.pcolormesh(X, Y, np.zeros_like(X), cmap='YlOrRd', alpha=0.5)\n",
    "    plt.colorbar(heatmap, label='Coverage Intensity')\n",
    "    \n",
    "    def update_heatmap(frame):\n",
    "        frame_data = play_tracking[play_tracking['frameId'] == frame]\n",
    "        Z = np.zeros_like(X)\n",
    "        \n",
    "        # Calculate coverage intensity\n",
    "        for _, player in frame_data.iterrows():\n",
    "            dx = X - player['x']\n",
    "            dy = Y - player['y']\n",
    "            Z += np.exp(-(dx**2 + dy**2) / 100)  # Gaussian coverage\n",
    "        \n",
    "        heatmap.set_array(Z.ravel())\n",
    "        return [heatmap]\n",
    "    \n",
    "    # Create animation\n",
    "    anim = animation.FuncAnimation(fig, update_heatmap, frames=frames,\n",
    "                                 interval=50, blit=True)\n",
    "    \n",
    "    # Save animation\n",
    "    writer = animation.PillowWriter(fps=20)\n",
    "    anim.save(save_path, writer=writer)\n",
    "    plt.close()\n",
    "\n",
    "# Load best play data for animation\n",
    "print(f\"Creating animations for {top_team}...\")\n",
    "\n",
    "# Get the most interesting play based on efficiency metrics\n",
    "best_play = efficiency_df.nlargest(1, 'path_efficiency')\n",
    "play_id = best_play['playId'].iloc[0]\n",
    "game_id = best_play['gameId'].iloc[0]\n",
    "\n",
    "# Load play tracking data\n",
    "play_tracking = tracking_df[\n",
    "    (tracking_df['gameId'] == game_id) & \n",
    "    (tracking_df['playId'] == play_id) &\n",
    "    (tracking_df['club'] == top_team)\n",
    "].sort_values('frameId')\n",
    "\n",
    "print(f\"\\nCreating animations for play {play_id}\")\n",
    "\n",
    "# Create defender movement animation\n",
    "defender_animation_path = os.path.join(os.path.dirname(os.getcwd()), \n",
    "                                     'animations', \n",
    "                                     f'{top_team}_play_{play_id}_movement.gif')\n",
    "create_defender_animation(play_tracking, defender_animation_path)\n",
    "\n",
    "# Create zone coverage animation\n",
    "coverage_animation_path = os.path.join(os.path.dirname(os.getcwd()), \n",
    "                                     'animations', \n",
    "                                     f'{top_team}_play_{play_id}_coverage.gif')\n",
    "create_zone_coverage_animation(play_tracking, coverage_animation_path)\n",
    "\n",
    "print(\"\\nAnimation Summary:\")\n",
    "print(f\"Defender Movement Animation: {defender_animation_path}\")\n",
    "print(f\"Zone Coverage Animation: {coverage_animation_path}\")\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 11: Annotated Animation Generation\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.animation as animation\n",
    "from matplotlib.patches import Circle, Rectangle, FancyArrowPatch\n",
    "from matplotlib.text import Annotation\n",
    "from tqdm import tqdm\n",
    "\n",
    "def create_annotated_animation(play_tracking, play_info, save_path):\n",
    "    \"\"\"Create annotated animation of defensive movement\"\"\"\n",
    "    fig, ax = create_field()\n",
    "    \n",
    "    # Prepare defender data\n",
    "    defenders = {}\n",
    "    trails = {}\n",
    "    for player_id in play_tracking['nflId'].unique():\n",
    "        player_data = play_tracking[play_tracking['nflId'] == player_id]\n",
    "        defenders[player_id] = player_data[['frameId', 'x', 'y', 's', 'dir']].values\n",
    "        trails[player_id] = []\n",
    "    \n",
    "    # Initialize plots\n",
    "    defender_plots = {}\n",
    "    trail_plots = {}\n",
    "    for player_id in defenders:\n",
    "        defender_plots[player_id], = ax.plot([], [], 'wo', markersize=10)\n",
    "        trail_plots[player_id], = ax.plot([], [], 'w-', alpha=0.3)\n",
    "    \n",
    "    # Add display elements\n",
    "    time_text = ax.text(5, 55, '', color='white', fontsize=12)\n",
    "    speed_text = ax.text(80, 55, '', color='white', fontsize=12)\n",
    "    \n",
    "    # Add annotation arrows and text boxes\n",
    "    annotations = []\n",
    "    def add_annotation(x, y, text, xytext_offset=(30, 30)):\n",
    "        ann = ax.annotate(\n",
    "            text,\n",
    "            xy=(x, y),\n",
    "            xytext=(x + xytext_offset[0], y + xytext_offset[1]),\n",
    "            color='white',\n",
    "            bbox=dict(facecolor='black', alpha=0.7),\n",
    "            arrowprops=dict(arrowstyle='->'),\n",
    "            animated=True\n",
    "        )\n",
    "        annotations.append(ann)\n",
    "        return ann\n",
    "    \n",
    "    def init():\n",
    "        for player_id in defenders:\n",
    "            defender_plots[player_id].set_data([], [])\n",
    "            trail_plots[player_id].set_data([], [])\n",
    "        time_text.set_text('')\n",
    "        speed_text.set_text('')\n",
    "        for ann in annotations:\n",
    "            ann.set_visible(False)\n",
    "        return (list(defender_plots.values()) + \n",
    "                list(trail_plots.values()) + \n",
    "                [time_text, speed_text] + \n",
    "                annotations)\n",
    "    \n",
    "    def animate(frame):\n",
    "        # Update defender positions and trails\n",
    "        max_speed = 0\n",
    "        for player_id in defenders:\n",
    "            if frame < len(defenders[player_id]):\n",
    "                x, y = defenders[player_id][frame][1:3]\n",
    "                speed = defenders[player_id][frame][3]\n",
    "                max_speed = max(max_speed, speed)\n",
    "                \n",
    "                defender_plots[player_id].set_data([x], [y])\n",
    "                \n",
    "                # Update trails\n",
    "                trails[player_id].append([x, y])\n",
    "                if len(trails[player_id]) > 10:\n",
    "                    trails[player_id].pop(0)\n",
    "                trail_data = np.array(trails[player_id])\n",
    "                trail_plots[player_id].set_data(trail_data[:, 0], trail_data[:, 1])\n",
    "        \n",
    "        # Update display texts\n",
    "        time_text.set_text(f'Frame: {frame}')\n",
    "        speed_text.set_text(f'Max Speed: {max_speed:.1f} yards/s')\n",
    "        \n",
    "        # Update annotations based on frame\n",
    "        for ann in annotations:\n",
    "            ann.set_visible(True)\n",
    "            \n",
    "        # Add frame-specific annotations\n",
    "        if frame == 0:\n",
    "            add_annotation(\n",
    "                defenders[list(defenders.keys())[0]][frame][1],\n",
    "                defenders[list(defenders.keys())[0]][frame][2],\n",
    "                'Starting Position'\n",
    "            )\n",
    "        elif frame == len(defenders[list(defenders.keys())[0]])//2:\n",
    "            add_annotation(\n",
    "                defenders[list(defenders.keys())[0]][frame][1],\n",
    "                defenders[list(defenders.keys())[0]][frame][2],\n",
    "                'Coverage Adjustment'\n",
    "            )\n",
    "        \n",
    "        return (list(defender_plots.values()) + \n",
    "                list(trail_plots.values()) + \n",
    "                [time_text, speed_text] + \n",
    "                annotations)\n",
    "    \n",
    "    # Create animation\n",
    "    anim = animation.FuncAnimation(fig, animate, init_func=init,\n",
    "                                 frames=len(defenders[list(defenders.keys())[0]]),\n",
    "                                 interval=50, blit=True)\n",
    "    \n",
    "    # Add title with play information\n",
    "    plt.title(f\"{top_team} Defensive Movement Analysis\\n\"\n",
    "             f\"Play ID: {play_info['playId']}, Game ID: {play_info['gameId']}\")\n",
    "    \n",
    "    # Save animation\n",
    "    writer = animation.PillowWriter(fps=20)\n",
    "    anim.save(save_path, writer=writer)\n",
    "    plt.close()\n",
    "\n",
    "# Create annotated version of the movement animation\n",
    "print(\"Creating annotated animation...\")\n",
    "\n",
    "# Get play information\n",
    "best_play_info = {\n",
    "    'playId': play_id,\n",
    "    'gameId': game_id\n",
    "}\n",
    "\n",
    "# Create annotated animation\n",
    "annotated_animation_path = os.path.join(os.path.dirname(os.getcwd()), \n",
    "                                      'animations', \n",
    "                                      f'{top_team}_play_{play_id}_annotated.gif')\n",
    "\n",
    "create_annotated_animation(play_tracking, best_play_info, annotated_animation_path)\n",
    "\n",
    "print(f\"\\nAnnotated animation saved to: {annotated_animation_path}\")\n",
    "print(f\"Final memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 12: Animation Documentation and Selection\n",
    "import pandas as pd\n",
    "import json\n",
    "import os\n",
    "from datetime import datetime\n",
    "import numpy as np\n",
    "\n",
    "# Add JSON encoder to handle numpy types\n",
    "class NumpyEncoder(json.JSONEncoder):\n",
    "    def default(self, obj):\n",
    "        if isinstance(obj, (np.int64, np.int32, np.int16, np.int8)):\n",
    "            return int(obj)\n",
    "        if isinstance(obj, (np.float64, np.float32)):\n",
    "            return float(obj)\n",
    "        if isinstance(obj, np.ndarray):\n",
    "            return obj.tolist()\n",
    "        return super(NumpyEncoder, self).default(obj)\n",
    "\n",
    "def create_animation_documentation(team, play_id, game_id, animation_type):\n",
    "    \"\"\"Create detailed documentation for each selected animation\"\"\"\n",
    "    \n",
    "    # Load relevant metrics for this play\n",
    "    efficiency_metrics = pd.read_csv(os.path.join(os.path.dirname(os.getcwd()), \n",
    "                                                'results', \n",
    "                                                f'{team}_efficiency_metrics.csv'))\n",
    "    play_metrics = efficiency_metrics[\n",
    "        (efficiency_metrics['playId'] == play_id) & \n",
    "        (efficiency_metrics['gameId'] == game_id)\n",
    "    ].iloc[0]\n",
    "    \n",
    "    # Load play information\n",
    "    plays_df = pd.read_csv(os.path.join(DATA_DIR, 'plays.csv'))\n",
    "    play_info = plays_df[\n",
    "        (plays_df['gameId'] == game_id) & \n",
    "        (plays_df['playId'] == play_id)\n",
    "    ].iloc[0]\n",
    "    \n",
    "    documentation = {\n",
    "        'animation_id': f\"{team}_play_{play_id}_{animation_type}\",\n",
    "        'team': team,\n",
    "        'play_id': int(play_id),  # Convert np.int64 to regular int\n",
    "        'game_id': int(game_id),  # Convert np.int64 to regular int\n",
    "        'animation_type': animation_type,\n",
    "        'metrics': {\n",
    "            'path_efficiency': float(play_metrics['path_efficiency']),\n",
    "            'avg_deviation': float(play_metrics['avg_deviation']),\n",
    "            'total_distance': float(play_metrics['total_distance'])\n",
    "        },\n",
    "        'play_context': {\n",
    "            'quarter': int(play_info['quarter']),\n",
    "            'down': int(play_info['down']),\n",
    "            'yards_to_go': int(play_info['yardsToGo']),\n",
    "            'play_description': str(play_info['playDescription'])\n",
    "        },\n",
    "        'key_features': {\n",
    "            'movement_patterns': [\n",
    "                \"Initial defensive alignment\",\n",
    "                \"Coverage adjustment response\",\n",
    "                \"Defender spacing maintenance\",\n",
    "                \"Route recognition reaction\",\n",
    "                \"Recovery movement patterns\"\n",
    "            ],\n",
    "            'technical_highlights': [\n",
    "                \"Speed variations during coverage\",\n",
    "                \"Directional changes\",\n",
    "                \"Zone handoff execution\",\n",
    "                \"Space maintenance discipline\"\n",
    "            ]\n",
    "        },\n",
    "        'markdown_description': f\"\"\"\n",
    "## Defensive Movement Analysis: {team} Play {play_id}\n",
    "\n",
    "### Play Context\n",
    "- Quarter: {int(play_info['quarter'])}\n",
    "- Down: {int(play_info['down'])}\n",
    "- Yards to Go: {int(play_info['yardsToGo'])}\n",
    "- Situation: {str(play_info['playDescription'])}\n",
    "\n",
    "### Key Defensive Features\n",
    "1. **Initial Formation**\n",
    "   - Defenders aligned in {play_info['defensiveTeam']} base defense\n",
    "   - Structured spacing creating optimal coverage zones\n",
    "\n",
    "2. **Movement Efficiency**\n",
    "   - Path Efficiency: {float(play_metrics['path_efficiency']):.2f}\n",
    "   - Average Deviation: {float(play_metrics['avg_deviation']):.2f} yards\n",
    "   - Total Distance: {float(play_metrics['total_distance']):.2f} yards\n",
    "\n",
    "3. **Coverage Execution**\n",
    "   - Synchronized defensive adjustments\n",
    "   - Maintained zone integrity throughout the play\n",
    "   - Effective space control and gap discipline\n",
    "\n",
    "### Technical Analysis\n",
    "- White dots represent defender positions\n",
    "- Trailing lines show recent movement paths\n",
    "- Speed indicators display instantaneous velocity\n",
    "- Coverage zones illustrated by position relationships\n",
    "\"\"\"\n",
    "    }\n",
    "    \n",
    "    return documentation\n",
    "\n",
    "# Document our selected animations\n",
    "selected_animations = [\n",
    "    {\n",
    "        'team': top_team,\n",
    "        'play_id': play_id,\n",
    "        'game_id': game_id,\n",
    "        'animation_type': 'movement'\n",
    "    }\n",
    "    # Add more animations here if we select others\n",
    "]\n",
    "\n",
    "# Create documentation for each selected animation\n",
    "animation_docs = []\n",
    "for anim in selected_animations:\n",
    "    doc = create_animation_documentation(\n",
    "        anim['team'],\n",
    "        anim['play_id'],\n",
    "        anim['game_id'],\n",
    "        anim['animation_type']\n",
    "    )\n",
    "    animation_docs.append(doc)\n",
    "\n",
    "# Save documentation using custom encoder\n",
    "docs_path = os.path.join(os.path.dirname(os.getcwd()), \n",
    "                        'results', \n",
    "                        'selected_animations_documentation.json')\n",
    "with open(docs_path, 'w') as f:\n",
    "    json.dump(animation_docs, f, indent=4, cls=NumpyEncoder)\n",
    "\n",
    "# Print markdown for each selected animation\n",
    "print(\"\\nAnimation Documentation for Kaggle Notebook:\")\n",
    "print(\"\\nNote: These descriptions will be used in the final markdown cells\")\n",
    "print(\"Each animation counts toward our 9-visualization limit\\n\")\n",
    "\n",
    "for doc in animation_docs:\n",
    "    print(doc['markdown_description'])\n",
    "    print(\"\\n\" + \"=\"*80 + \"\\n\")\n",
    "\n",
    "print(f\"\\nDocumentation saved to: {docs_path}\")\n",
    "print(f\"Final memory usage: {get_memory_usage():.2f} MB\")\n",
    "\n",
    "# Create a visualization count tracker\n",
    "viz_count = {\n",
    "    'plots': len(os.listdir(os.path.join(os.path.dirname(os.getcwd()), 'plots'))),\n",
    "    'animations': len(selected_animations),\n",
    "    'total': 0\n",
    "}\n",
    "viz_count['total'] = viz_count['plots'] + viz_count['animations']\n",
    "\n",
    "print(\"\\nVisualization Count:\")\n",
    "print(f\"Plots: {viz_count['plots']}\")\n",
    "print(f\"Animations: {viz_count['animations']}\")\n",
    "print(f\"Total: {viz_count['total']}/9 allowed visualizations\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 13: Final Visualization Selection and Organization\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import shutil\n",
    "from datetime import datetime\n",
    "\n",
    "def analyze_visualization_impact(viz_type, filename, metrics_df):\n",
    "    \"\"\"Analyze the significance of each visualization\"\"\"\n",
    "    if viz_type == 'plot':\n",
    "        # Extract play_id from filename if present\n",
    "        play_id = filename.split('_play_')[-1].split('_')[0] if '_play_' in filename else None\n",
    "        \n",
    "        if play_id and play_id.isdigit():\n",
    "            # Get metrics for this play\n",
    "            play_metrics = metrics_df[metrics_df['playId'] == int(play_id)]\n",
    "            if not play_metrics.empty:\n",
    "                return {\n",
    "                    'filename': filename,\n",
    "                    'type': viz_type,\n",
    "                    'impact_score': float(play_metrics['path_efficiency'].iloc[0]),\n",
    "                    'context': 'Play-specific visualization'\n",
    "                }\n",
    "        \n",
    "        # For non-play-specific plots\n",
    "        return {\n",
    "            'filename': filename,\n",
    "            'type': viz_type,\n",
    "            'impact_score': 1.0 if 'summary' in filename else 0.5,\n",
    "            'context': 'Summary visualization' if 'summary' in filename else 'Supporting visualization'\n",
    "        }\n",
    "    \n",
    "    else:  # animation\n",
    "        return {\n",
    "            'filename': filename,\n",
    "            'type': 'animation',\n",
    "            'impact_score': 1.0,  # Animations are typically high-impact\n",
    "            'context': 'Key defensive movement visualization'\n",
    "        }\n",
    "\n",
    "def select_final_visualizations(top_team, max_viz=9):\n",
    "    \"\"\"Select the most impactful visualizations within the limit\"\"\"\n",
    "    # Load metrics for reference\n",
    "    efficiency_df = pd.read_csv(os.path.join(os.path.dirname(os.getcwd()), \n",
    "                                           'results', \n",
    "                                           f'{top_team}_efficiency_metrics.csv'))\n",
    "    \n",
    "    # Get all visualizations\n",
    "    plots_dir = os.path.join(os.path.dirname(os.getcwd()), 'plots')\n",
    "    animations_dir = os.path.join(os.path.dirname(os.getcwd()), 'animations')\n",
    "    \n",
    "    all_visualizations = []\n",
    "    \n",
    "    # Analyze plots\n",
    "    for filename in os.listdir(plots_dir):\n",
    "        if filename.endswith(('.png', '.jpg')):\n",
    "            analysis = analyze_visualization_impact('plot', filename, efficiency_df)\n",
    "            all_visualizations.append(analysis)\n",
    "    \n",
    "    # Analyze animations\n",
    "    for filename in os.listdir(animations_dir):\n",
    "        if filename.endswith('.gif'):\n",
    "            analysis = analyze_visualization_impact('animation', filename, efficiency_df)\n",
    "            all_visualizations.append(analysis)\n",
    "    \n",
    "    # Sort by impact score\n",
    "    sorted_viz = sorted(all_visualizations, \n",
    "                       key=lambda x: x['impact_score'], \n",
    "                       reverse=True)\n",
    "    \n",
    "    # Select top visualizations while ensuring we have key categories\n",
    "    final_selection = []\n",
    "    categories_needed = {\n",
    "        'summary': 1,\n",
    "        'animation': 1,\n",
    "        'play_specific': 3\n",
    "    }\n",
    "    \n",
    "    for viz in sorted_viz:\n",
    "        if len(final_selection) >= max_viz:\n",
    "            break\n",
    "            \n",
    "        # Ensure we have necessary category representation\n",
    "        if 'summary' in viz['context'].lower() and categories_needed['summary'] > 0:\n",
    "            final_selection.append(viz)\n",
    "            categories_needed['summary'] -= 1\n",
    "        elif viz['type'] == 'animation' and categories_needed['animation'] > 0:\n",
    "            final_selection.append(viz)\n",
    "            categories_needed['animation'] -= 1\n",
    "        elif 'play-specific' in viz['context'].lower() and categories_needed['play_specific'] > 0:\n",
    "            final_selection.append(viz)\n",
    "            categories_needed['play_specific'] -= 1\n",
    "        elif sum(categories_needed.values()) < max_viz - len(final_selection):\n",
    "            final_selection.append(viz)\n",
    "    \n",
    "    return final_selection\n",
    "\n",
    "def create_visualization_manifest(selected_viz):\n",
    "    \"\"\"Create a markdown-ready manifest of selected visualizations\"\"\"\n",
    "    manifest = \"# Selected Visualizations\\n\\n\"\n",
    "    \n",
    "    # Group by type\n",
    "    plots = [v for v in selected_viz if v['type'] == 'plot']\n",
    "    animations = [v for v in selected_viz if v['type'] == 'animation']\n",
    "    \n",
    "    # Add plots section\n",
    "    manifest += \"## Static Visualizations\\n\\n\"\n",
    "    for i, plot in enumerate(plots, 1):\n",
    "        manifest += f\"{i}. **{plot['filename']}**\\n\"\n",
    "        manifest += f\"   - Context: {plot['context']}\\n\"\n",
    "        manifest += f\"   - Impact Score: {plot['impact_score']:.2f}\\n\\n\"\n",
    "    \n",
    "    # Add animations section\n",
    "    manifest += \"## Animations\\n\\n\"\n",
    "    for i, anim in enumerate(animations, 1):\n",
    "        manifest += f\"{i}. **{anim['filename']}**\\n\"\n",
    "        manifest += f\"   - Context: {anim['context']}\\n\"\n",
    "        manifest += f\"   - Impact Score: {anim['impact_score']:.2f}\\n\\n\"\n",
    "    \n",
    "    return manifest\n",
    "\n",
    "# Execute visualization selection\n",
    "print(\"Selecting final visualizations...\")\n",
    "final_visualizations = select_final_visualizations(top_team)\n",
    "\n",
    "# Create and save manifest\n",
    "manifest = create_visualization_manifest(final_visualizations)\n",
    "manifest_path = os.path.join(os.path.dirname(os.getcwd()), \n",
    "                            'results', \n",
    "                            'final_visualization_manifest.md')\n",
    "with open(manifest_path, 'w') as f:\n",
    "    f.write(manifest)\n",
    "\n",
    "# Create directory for final selections\n",
    "final_dir = os.path.join(os.path.dirname(os.getcwd()), 'final_visualizations')\n",
    "os.makedirs(final_dir, exist_ok=True)\n",
    "\n",
    "# Copy selected visualizations to final directory\n",
    "for viz in final_visualizations:\n",
    "    source_dir = os.path.join(os.path.dirname(os.getcwd()),\n",
    "                             'plots' if viz['type'] == 'plot' else 'animations')\n",
    "    source_path = os.path.join(source_dir, viz['filename'])\n",
    "    dest_path = os.path.join(final_dir, viz['filename'])\n",
    "    shutil.copy2(source_path, dest_path)\n",
    "\n",
    "print(\"\\nFinal Visualization Selection:\")\n",
    "print(f\"Total selected: {len(final_visualizations)}/9\")\n",
    "print(\"\\nVisualization manifest saved to:\", manifest_path)\n",
    "print(\"Selected visualizations copied to:\", final_dir)\n",
    "print(\"\\nManifest Preview:\")\n",
    "print(\"=\"*80)\n",
    "print(manifest[:500] + \"...\\n\")\n",
    "print(\"=\"*80)\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 14 (Revised): Kaggle Notebook Preparation with Mathematical Foundation\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import os\n",
    "from datetime import datetime\n",
    "\n",
    "def create_notebook_sections():\n",
    "    \"\"\"Create structured markdown content for Kaggle notebook with mathematical detail\"\"\"\n",
    "    \n",
    "    # Load previous content\n",
    "    with open(os.path.join(os.path.dirname(os.getcwd()), \n",
    "                          'results', \n",
    "                          'final_visualization_manifest.md'), 'r') as f:\n",
    "        viz_manifest = f.read()\n",
    "    \n",
    "    with open(os.path.join(os.path.dirname(os.getcwd()),\n",
    "                          'results',\n",
    "                          'selected_animations_documentation.json'), 'r') as f:\n",
    "        animation_docs = json.load(f)\n",
    "\n",
    "    notebook_content = {\n",
    "        'introduction': f\"\"\"# Ecological Movement Modeling in NFL Defense: A Mathematical Analysis\n",
    "        \n",
    "## Introduction\n",
    "This analysis explores defensive movement patterns in the NFL through the lens of ecological \n",
    "movement modeling, focusing on the {top_team}'s defensive unit. By applying principles from \n",
    "ecological mathematics and movement theory, we uncover the underlying patterns that govern \n",
    "effective defensive coverage.\n",
    "\n",
    "The movement of defensive players can be understood as a complex dynamical system, where \n",
    "individual agents (defenders) respond to both local and global stimuli while maintaining \n",
    "coordinated coverage of spatial zones.\n",
    "\"\"\",\n",
    "        \n",
    "        'mathematical_foundation': \"\"\"## Mathematical Framework\n",
    "\n",
    "### 1. Movement Dynamics\n",
    "Individual defender movement can be modeled using a stochastic differential equation:\n",
    "\n",
    "$$ \\\\frac{dx}{dt} = v(t) + \\\\sigma(x,t)\\\\xi(t) $$\n",
    "\n",
    "Where:\n",
    "- $x(t)$ represents position vector\n",
    "- $v(t)$ is the velocity field\n",
    "- $\\\\sigma(x,t)$ captures environmental influence\n",
    "- $\\\\xi(t)$ represents random fluctuations\n",
    "\n",
    "### 2. Coverage Zone Optimization\n",
    "The optimal coverage configuration minimizes the exposure function:\n",
    "\n",
    "$$ E(\\\\mathbf{X}) = \\\\int_\\\\Omega \\\\min_{i} \\\\|x - x_i\\\\| dx $$\n",
    "\n",
    "Where:\n",
    "- $\\\\mathbf{X} = (x_1,...,x_n)$ represents defender positions\n",
    "- $\\\\Omega$ is the field area\n",
    "- $\\\\|x - x_i\\\\|$ is the distance to defender $i$\n",
    "\n",
    "### 3. Collective Motion Metrics\n",
    "Team coordination is quantified through the order parameter:\n",
    "\n",
    "$$ \\\\psi(t) = \\\\frac{1}{N}\\\\left|\\\\sum_{i=1}^N e^{i\\\\theta_i(t)}\\\\right| $$\n",
    "\n",
    "Where:\n",
    "- $N$ is the number of defenders\n",
    "- $\\\\theta_i(t)$ is the movement direction of defender $i$\n",
    "\n",
    "### 4. Path Efficiency Analysis\n",
    "Movement efficiency is calculated using the ratio:\n",
    "\n",
    "$$ \\\\eta = \\\\frac{\\\\|x_f - x_0\\\\|}{\\\\int_0^T \\\\|v(t)\\\\| dt} $$\n",
    "\n",
    "Where:\n",
    "- $x_0$ and $x_f$ are initial and final positions\n",
    "- $T$ is the play duration\n",
    "- $\\\\|v(t)\\\\|$ is instantaneous speed\n",
    "\n",
    "### 5. Zone Coverage Model\n",
    "Coverage quality is evaluated using a potential field approach:\n",
    "\n",
    "$$ \\\\phi(x) = \\\\sum_{i=1}^N A_i e^{-\\\\|x-x_i\\\\|^2/2\\\\sigma_i^2} $$\n",
    "\n",
    "Where:\n",
    "- $A_i$ is defender effectiveness\n",
    "- $\\\\sigma_i$ is coverage radius\n",
    "\"\"\",\n",
    "\n",
    "        'methodology': f\"\"\"## Methodology: From Theory to Practice\n",
    "\n",
    "### Implementation of Mathematical Models\n",
    "Our analysis translates these mathematical principles into practical metrics:\n",
    "\n",
    "1. **Movement Pattern Analysis**\n",
    "   - Discretized version of continuous movement equations\n",
    "   - Numerical integration of path efficiencies\n",
    "   - Statistical analysis of coverage patterns\n",
    "\n",
    "2. **Spatial Analysis Framework**\n",
    "   - Voronoi tessellation of defensive coverage\n",
    "   - Dynamic time warping for pattern matching\n",
    "   - Kernel density estimation for zone coverage\n",
    "\n",
    "3. **Efficiency Calculations**\n",
    "   - Path integral computation\n",
    "   - Coverage optimization algorithms\n",
    "   - Coordination metric evaluation\n",
    "\n",
    "### Data Processing Pipeline\n",
    "1. Position tracking at 10 Hz\n",
    "2. Velocity and acceleration computation\n",
    "3. Coverage zone calculation\n",
    "4. Pattern recognition and classification\n",
    "\"\"\",\n",
    "        \n",
    "        'movement_analysis': \"\"\"## Defensive Movement Analysis\n",
    "\n",
    "### Individual Defender Dynamics\n",
    "We observe that defender movements follow a modified Ornstein-Uhlenbeck process:\n",
    "\n",
    "$$ dv = -\\\\gamma(v - v_p)dt + \\\\sigma dW_t $$\n",
    "\n",
    "Where:\n",
    "- $v_p$ is preferred velocity\n",
    "- $\\\\gamma$ is response rate\n",
    "- $W_t$ is a Wiener process\n",
    "\n",
    "This mathematical framework reveals how defenders balance between:\n",
    "1. Maintaining assigned coverage responsibilities\n",
    "2. Responding to offensive movements\n",
    "3. Coordinating with teammates\n",
    "\n",
    "[Insert Movement Pattern Visualization 1]\n",
    "\n",
    "The animation above demonstrates these principles in action, showing how theoretical\n",
    "predictions align with actual defensive movements.\n",
    "\"\"\",\n",
    "\n",
    "        'coverage_analysis': \"\"\"## Coverage Zone Analysis\n",
    "\n",
    "### Optimal Coverage Theory\n",
    "The defensive unit seeks to minimize the coverage gap function:\n",
    "\n",
    "$$ G(\\\\mathbf{X}) = \\\\max_{y \\\\in \\\\Omega} \\\\min_{i} \\\\|y - x_i\\\\| $$\n",
    "\n",
    "This optimization problem leads to emergent behavior that we observe in the tracking data:\n",
    "\n",
    "[Insert Coverage Zone Visualization 2]\n",
    "\n",
    "The heat map above shows the realized coverage density compared to theoretical optimal\n",
    "distribution.\n",
    "\"\"\",\n",
    "\n",
    "        'efficiency_metrics': \"\"\"## Efficiency and Performance Metrics\n",
    "\n",
    "### Quantitative Performance Measures\n",
    "Movement efficiency is evaluated through multiple metrics:\n",
    "\n",
    "1. **Path Efficiency Ratio**:\n",
    "   $$ \\\\eta_{path} = \\\\frac{d_{optimal}}{d_{actual}} $$\n",
    "\n",
    "2. **Coverage Quality Index**:\n",
    "   $$ Q_c = \\\\int_\\\\Omega \\\\phi(x)dx $$\n",
    "\n",
    "3. **Coordination Coefficient**:\n",
    "   $$ C = \\\\frac{1}{T}\\\\int_0^T \\\\psi(t)dt $$\n",
    "\n",
    "[Insert Efficiency Comparison Visualization 3]\n",
    "\"\"\",\n",
    "\n",
    "        'results_interpretation': \"\"\"## Results and Interpretation\n",
    "\n",
    "Our mathematical analysis reveals several key insights:\n",
    "\n",
    "1. **Optimality Conditions**\n",
    "   The observed defensive movements satisfy the Euler-Lagrange equation:\n",
    "   $$ \\\\frac{d}{dt}\\\\frac{\\\\partial L}{\\\\partial \\\\dot{x}} - \\\\frac{\\\\partial L}{\\\\partial x} = 0 $$\n",
    "   where $L$ is the Lagrangian of the system.\n",
    "\n",
    "2. **Pattern Emergence**\n",
    "   Defensive coordination emerges from local rules following:\n",
    "   $$ \\\\dot{x}_i = \\\\sum_{j \\\\neq i} f(\\\\|x_j - x_i\\\\|)(x_j - x_i) $$\n",
    "\n",
    "3. **Efficiency Boundaries**\n",
    "   Performance limits are governed by:\n",
    "   $$ \\\\eta_{max} = \\\\sup_{\\\\mathbf{X}} \\\\{\\\\eta(\\\\mathbf{X}) : C(\\\\mathbf{X}) \\\\geq c_{min}\\\\} $$\n",
    "\n",
    "[Insert Pattern Analysis Visualization 4]\n",
    "\"\"\",\n",
    "\n",
    "        'conclusions': \"\"\"## Conclusions and Future Directions\n",
    "\n",
    "Our mathematical analysis demonstrates that NFL defensive movements follow predictable\n",
    "patterns governed by fundamental principles of collective motion and optimal control theory.\n",
    "Key findings include:\n",
    "\n",
    "1. Emergence of optimal coverage patterns through local interactions\n",
    "2. Quantifiable trade-offs between individual and collective efficiency\n",
    "3. Predictable response patterns based on environmental constraints\n",
    "\n",
    "Future work will explore:\n",
    "- Higher-order movement correlations\n",
    "- Non-linear response functions\n",
    "- Stochastic game theory applications\n",
    "\"\"\",\n",
    "\n",
    "        'github_reference': \"\"\"## Extended Mathematical Analysis\n",
    "\n",
    "Complete mathematical derivations, numerical methods, and additional analyses are available\n",
    "in our GitHub repository. This includes:\n",
    "\n",
    "1. Detailed mathematical proofs\n",
    "2. Numerical simulation code\n",
    "3. Statistical validation tests\n",
    "4. Extended data visualizations\n",
    "\n",
    "[GitHub Repository Link to be added]\n",
    "\"\"\"\n",
    "    }\n",
    "    \n",
    "    return notebook_content\n",
    "\n",
    "# Generate notebook content\n",
    "print(\"Generating enhanced Kaggle notebook content...\")\n",
    "notebook_sections = create_notebook_sections()\n",
    "\n",
    "# Save sections to markdown files\n",
    "output_dir = os.path.join(os.path.dirname(os.getcwd()), 'kaggle_notebook')\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "\n",
    "for section_name, content in notebook_sections.items():\n",
    "    file_path = os.path.join(output_dir, f'{section_name}.md')\n",
    "    with open(file_path, 'w') as f:\n",
    "        f.write(content)\n",
    "\n",
    "# Create word count summary\n",
    "word_counts = {\n",
    "    section: len(content.split()) \n",
    "    for section, content in notebook_sections.items()\n",
    "}\n",
    "total_words = sum(word_counts.values())\n",
    "\n",
    "print(\"\\nNotebook Section Word Counts:\")\n",
    "for section, count in word_counts.items():\n",
    "    print(f\"{section}: {count} words\")\n",
    "print(f\"\\nTotal Words: {total_words}/2000\")\n",
    "\n",
    "print(\"\\nNote: LaTeX equations not counted in word count\")\n",
    "print(f\"\\nNotebook sections saved to: {output_dir}\")\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 15: Visualization Review and Selection\n",
    "import os\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "import json\n",
    "\n",
    "def review_visualizations():\n",
    "    \"\"\"Review and catalog all generated visualizations\"\"\"\n",
    "    \n",
    "    plots_dir = os.path.join(os.path.dirname(os.getcwd()), 'plots')\n",
    "    animations_dir = os.path.join(os.path.dirname(os.getcwd()), 'animations')\n",
    "    final_dir = os.path.join(os.path.dirname(os.getcwd()), 'final_visualizations')\n",
    "\n",
    "    # Create catalog of all visualizations\n",
    "    catalog = {\n",
    "        'plots': [],\n",
    "        'animations': [],\n",
    "        'selected': []\n",
    "    }\n",
    "\n",
    "    # Review plots\n",
    "    print(\"\\nAvailable Plots:\")\n",
    "    print(\"-\" * 80)\n",
    "    for filename in sorted(os.listdir(plots_dir)):\n",
    "        if filename.endswith(('.png', '.jpg')):\n",
    "            file_path = os.path.join(plots_dir, filename)\n",
    "            img = Image.open(file_path)\n",
    "            size_mb = os.path.getsize(file_path) / (1024 * 1024)\n",
    "            \n",
    "            catalog['plots'].append({\n",
    "                'filename': filename,\n",
    "                'dimensions': img.size,\n",
    "                'size_mb': size_mb,\n",
    "                'type': 'plot'\n",
    "            })\n",
    "            \n",
    "            print(f\"Plot: {filename}\")\n",
    "            print(f\"Dimensions: {img.size}\")\n",
    "            print(f\"Size: {size_mb:.2f} MB\")\n",
    "            print(\"-\" * 80)\n",
    "\n",
    "    # Review animations\n",
    "    print(\"\\nAvailable Animations:\")\n",
    "    print(\"-\" * 80)\n",
    "    for filename in sorted(os.listdir(animations_dir)):\n",
    "        if filename.endswith('.gif'):\n",
    "            file_path = os.path.join(animations_dir, filename)\n",
    "            size_mb = os.path.getsize(file_path) / (1024 * 1024)\n",
    "            \n",
    "            catalog['animations'].append({\n",
    "                'filename': filename,\n",
    "                'size_mb': size_mb,\n",
    "                'type': 'animation'\n",
    "            })\n",
    "            \n",
    "            print(f\"Animation: {filename}\")\n",
    "            print(f\"Size: {size_mb:.2f} MB\")\n",
    "            print(\"-\" * 80)\n",
    "\n",
    "    # Review current selections\n",
    "    if os.path.exists(final_dir):\n",
    "        print(\"\\nCurrently Selected Visualizations:\")\n",
    "        print(\"-\" * 80)\n",
    "        for filename in sorted(os.listdir(final_dir)):\n",
    "            file_path = os.path.join(final_dir, filename)\n",
    "            size_mb = os.path.getsize(file_path) / (1024 * 1024)\n",
    "            \n",
    "            catalog['selected'].append({\n",
    "                'filename': filename,\n",
    "                'size_mb': size_mb\n",
    "            })\n",
    "            \n",
    "            print(f\"Selected: {filename}\")\n",
    "            print(f\"Size: {size_mb:.2f} MB\")\n",
    "            print(\"-\" * 80)\n",
    "\n",
    "    return catalog\n",
    "\n",
    "def categorize_visualizations(catalog):\n",
    "    \"\"\"Categorize visualizations by their content type\"\"\"\n",
    "    categories = {\n",
    "        'summary_plots': [],\n",
    "        'movement_analysis': [],\n",
    "        'coverage_analysis': [],\n",
    "        'efficiency_analysis': [],\n",
    "        'animations': []\n",
    "    }\n",
    "    \n",
    "    for item in catalog['plots']:\n",
    "        filename = item['filename']\n",
    "        if 'summary' in filename:\n",
    "            categories['summary_plots'].append(item)\n",
    "        elif 'movement' in filename or 'pattern' in filename:\n",
    "            categories['movement_analysis'].append(item)\n",
    "        elif 'coverage' in filename:\n",
    "            categories['coverage_analysis'].append(item)\n",
    "        elif 'efficiency' in filename:\n",
    "            categories['efficiency_analysis'].append(item)\n",
    "    \n",
    "    categories['animations'] = catalog['animations']\n",
    "    \n",
    "    return categories\n",
    "\n",
    "# Execute review\n",
    "print(\"Reviewing all visualizations...\")\n",
    "catalog = review_visualizations()\n",
    "categories = categorize_visualizations(catalog)\n",
    "\n",
    "# Print summary by category\n",
    "print(\"\\nVisualization Summary by Category:\")\n",
    "print(\"=\" * 80)\n",
    "for category, items in categories.items():\n",
    "    print(f\"\\n{category.replace('_', ' ').title()}:\")\n",
    "    for item in items:\n",
    "        print(f\"- {item['filename']} ({item['size_mb']:.2f} MB)\")\n",
    "\n",
    "# Check Kaggle constraints\n",
    "total_selected = len(catalog['selected'])\n",
    "print(\"\\nKaggle Constraints Check:\")\n",
    "print(f\"Current selections: {total_selected}/9 allowed visualizations\")\n",
    "\n",
    "# Save catalog for reference\n",
    "catalog_path = os.path.join(os.path.dirname(os.getcwd()), \n",
    "                           'results', \n",
    "                           'visualization_catalog.json')\n",
    "with open(catalog_path, 'w') as f:\n",
    "    json.dump(catalog, f, indent=4)\n",
    "\n",
    "print(f\"\\nVisualization catalog saved to: {catalog_path}\")\n",
    "print(f\"Final memory usage: {get_memory_usage():.2f} MB\")\n",
    "\n",
    "# Provide recommendations\n",
    "print(\"\\nRecommended Visualization Selection:\")\n",
    "print(\"=\" * 80)\n",
    "print(\"For a coherent narrative, we should include:\")\n",
    "print(\"1. One overall summary visualization\")\n",
    "print(\"2. One key defensive movement animation\")\n",
    "print(\"3. One coverage pattern analysis\")\n",
    "print(\"4. One efficiency comparison\")\n",
    "print(\"5. One team-specific analysis\")\n",
    "print(\"\\nWould you like to review any specific category in detail?\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 16: Visualization Cleanup and Documentation Update\n",
    "import os\n",
    "import shutil\n",
    "import json\n",
    "\n",
    "def cleanup_visualizations():\n",
    "    \"\"\"Remove unnecessary visualizations and update documentation\"\"\"\n",
    "    final_dir = os.path.join(os.path.dirname(os.getcwd()), 'final_visualizations')\n",
    "    \n",
    "    # Files to remove\n",
    "    remove_files = [\n",
    "        'SF_play_467_movement.gif',\n",
    "        'SF_play_1992_formation.png'\n",
    "    ]\n",
    "    \n",
    "    # Remove files\n",
    "    removed_count = 0\n",
    "    for file in remove_files:\n",
    "        file_path = os.path.join(final_dir, file)\n",
    "        if os.path.exists(file_path):\n",
    "            os.remove(file_path)\n",
    "            removed_count += 1\n",
    "            print(f\"Removed: {file}\")\n",
    "    \n",
    "    # Count remaining visualizations\n",
    "    remaining_files = os.listdir(final_dir)\n",
    "    print(f\"\\nRemaining visualizations: {len(remaining_files)}/9\")\n",
    "    \n",
    "    return remaining_files\n",
    "\n",
    "def update_documentation():\n",
    "    \"\"\"Update all documentation files to reflect changes\"\"\"\n",
    "    # Update visualization manifest\n",
    "    manifest_path = os.path.join(os.path.dirname(os.getcwd()), \n",
    "                                'results', \n",
    "                                'final_visualization_manifest.md')\n",
    "    \n",
    "    # Load current manifest\n",
    "    with open(manifest_path, 'r') as f:\n",
    "        manifest_content = f.read()\n",
    "    \n",
    "    # Remove references to deleted files\n",
    "    updated_manifest = []\n",
    "    for line in manifest_content.split('\\n'):\n",
    "        if not any(removed in line for removed in ['467_movement.gif', '1992_formation.png']):\n",
    "            updated_manifest.append(line)\n",
    "    \n",
    "    # Save updated manifest\n",
    "    with open(manifest_path, 'w') as f:\n",
    "        f.write('\\n'.join(updated_manifest))\n",
    "    \n",
    "    # Update Kaggle notebook sections\n",
    "    notebook_dir = os.path.join(os.path.dirname(os.getcwd()), 'kaggle_notebook')\n",
    "    for section_file in os.listdir(notebook_dir):\n",
    "        if section_file.endswith('.md'):\n",
    "            file_path = os.path.join(notebook_dir, section_file)\n",
    "            with open(file_path, 'r') as f:\n",
    "                content = f.read()\n",
    "            \n",
    "            # Remove references to deleted files\n",
    "            updated_content = []\n",
    "            for line in content.split('\\n'):\n",
    "                if not any(removed in line for removed in ['467_movement.gif', '1992_formation.png']):\n",
    "                    updated_content.append(line)\n",
    "            \n",
    "            with open(file_path, 'w') as f:\n",
    "                f.write('\\n'.join(updated_content))\n",
    "\n",
    "    print(\"Documentation updated successfully\")\n",
    "\n",
    "def print_visualization_status():\n",
    "    \"\"\"Print current status of visualizations\"\"\"\n",
    "    final_dir = os.path.join(os.path.dirname(os.getcwd()), 'final_visualizations')\n",
    "    files = sorted(os.listdir(final_dir))\n",
    "    \n",
    "    print(\"\\nCurrent Visualizations:\")\n",
    "    print(\"=\" * 80)\n",
    "    for i, file in enumerate(files, 1):\n",
    "        file_path = os.path.join(final_dir, file)\n",
    "        size_mb = os.path.getsize(file_path) / (1024 * 1024)\n",
    "        print(f\"{i}. {file} ({size_mb:.2f} MB)\")\n",
    "\n",
    "# Execute cleanup\n",
    "print(\"Cleaning up visualizations...\")\n",
    "remaining_files = cleanup_visualizations()\n",
    "\n",
    "# Update documentation\n",
    "print(\"\\nUpdating documentation...\")\n",
    "update_documentation()\n",
    "\n",
    "# Show current status\n",
    "print_visualization_status()\n",
    "\n",
    "# Calculate remaining slots\n",
    "remaining_slots = 9 - len(remaining_files)\n",
    "print(f\"\\nRemaining visualization slots: {remaining_slots}\")\n",
    "\n",
    "if remaining_slots > 0:\n",
    "    print(\"\\nRecommendations for remaining slots:\")\n",
    "    print(\"1. Consider adding another coverage analysis visualization\")\n",
    "    print(\"2. Could include an additional efficiency metric visualization\")\n",
    "    print(\"3. Might add a comparative analysis visualization\")\n",
    "    print(\"\\nWould you like to review candidates for the remaining slots?\")\n",
    "\n",
    "print(f\"\\nFinal memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell 17: Create Comparative Analysis Visualization\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from matplotlib.gridspec import GridSpec\n",
    "from scipy import stats\n",
    "\n",
    "def create_comparative_analysis():\n",
    "    \"\"\"Create comprehensive comparative analysis of defensive performance\"\"\"\n",
    "    # Load efficiency metrics for all teams\n",
    "    metrics_by_team = {}\n",
    "    for team in NFC_WEST:\n",
    "        try:\n",
    "            file_path = os.path.join(os.path.dirname(os.getcwd()), \n",
    "                                   'results', \n",
    "                                   f'{team}_efficiency_metrics.csv')\n",
    "            if os.path.exists(file_path):\n",
    "                metrics_by_team[team] = pd.read_csv(file_path)\n",
    "        except:\n",
    "            print(f\"No metrics file found for {team}\")\n",
    "    \n",
    "    # Create comparison figure\n",
    "    plt.style.use('dark_background')\n",
    "    fig = plt.figure(figsize=(15, 10))\n",
    "    gs = GridSpec(2, 2, figure=fig)\n",
    "    \n",
    "    # 1. Efficiency Comparison (top left)\n",
    "    ax1 = fig.add_subplot(gs[0, 0])\n",
    "    efficiency_data = {team: metrics['path_efficiency'].mean() \n",
    "                      for team, metrics in metrics_by_team.items()}\n",
    "    efficiency_std = {team: metrics['path_efficiency'].std() \n",
    "                     for team, metrics in metrics_by_team.items()}\n",
    "    \n",
    "    teams = list(efficiency_data.keys())\n",
    "    values = list(efficiency_data.values())\n",
    "    std_values = list(efficiency_std.values())\n",
    "    \n",
    "    bars = ax1.bar(teams, values, yerr=std_values, \n",
    "                   alpha=0.7, capsize=5)\n",
    "    bars[teams.index(top_team)].set_color('gold')  # Highlight top team\n",
    "    ax1.set_title('Path Efficiency by Team')\n",
    "    ax1.set_ylabel('Average Path Efficiency')\n",
    "    \n",
    "    # 2. Coverage Area Comparison (top right)\n",
    "    ax2 = fig.add_subplot(gs[0, 1])\n",
    "    coverage_data = {\n",
    "        team: np.sqrt(metrics['total_distance'].mean()) # Using sqrt for better visualization\n",
    "        for team, metrics in metrics_by_team.items()\n",
    "    }\n",
    "    \n",
    "    teams = list(coverage_data.keys())\n",
    "    values = list(coverage_data.values())\n",
    "    \n",
    "    bars = ax2.bar(teams, values, alpha=0.7)\n",
    "    bars[teams.index(top_team)].set_color('gold')\n",
    "    ax2.set_title('Coverage Area by Team')\n",
    "    ax2.set_ylabel('Sqrt of Average Coverage (yards)')\n",
    "    \n",
    "    # 3. Movement Pattern Distribution (bottom left)\n",
    "    ax3 = fig.add_subplot(gs[1, 0])\n",
    "    pattern_data = []\n",
    "    for team in metrics_by_team:\n",
    "        metrics = metrics_by_team[team]\n",
    "        kernel = stats.gaussian_kde(metrics['avg_deviation'])\n",
    "        x_range = np.linspace(metrics['avg_deviation'].min(), \n",
    "                            metrics['avg_deviation'].max(), 100)\n",
    "        pattern_data.append((x_range, kernel(x_range)))\n",
    "    \n",
    "    for i, team in enumerate(metrics_by_team):\n",
    "        x, y = pattern_data[i]\n",
    "        ax3.plot(x, y, label=team, alpha=0.7)\n",
    "    ax3.set_title('Movement Pattern Distribution')\n",
    "    ax3.set_xlabel('Average Deviation (yards)')\n",
    "    ax3.set_ylabel('Density')\n",
    "    ax3.legend()\n",
    "    \n",
    "    # 4. Radar Chart of Key Metrics (bottom right)\n",
    "    ax4 = fig.add_subplot(gs[1, 1], projection='polar')\n",
    "    metrics_for_radar = ['path_efficiency', 'avg_deviation', 'total_distance']\n",
    "    \n",
    "    # Normalize metrics for radar chart\n",
    "    radar_data = {}\n",
    "    for team in metrics_by_team:\n",
    "        metrics = metrics_by_team[team]\n",
    "        radar_data[team] = [\n",
    "            metrics['path_efficiency'].mean() / max(m['path_efficiency'].mean() \n",
    "                for m in metrics_by_team.values()),\n",
    "            1 - (metrics['avg_deviation'].mean() / max(m['avg_deviation'].mean() \n",
    "                for m in metrics_by_team.values())),  # Invert for better visualization\n",
    "            metrics['total_distance'].mean() / max(m['total_distance'].mean() \n",
    "                for m in metrics_by_team.values())\n",
    "        ]\n",
    "    \n",
    "    angles = np.linspace(0, 2*np.pi, len(metrics_for_radar), endpoint=False)\n",
    "    \n",
    "    for team in radar_data:\n",
    "        values = radar_data[team]\n",
    "        values += values[:1]  # Complete the polygon\n",
    "        angles_plot = np.concatenate((angles, [angles[0]]))  # Complete the polygon\n",
    "        \n",
    "        ax4.plot(angles_plot, values, label=team, alpha=0.7)\n",
    "        ax4.fill(angles_plot, values, alpha=0.1)\n",
    "    \n",
    "    ax4.set_xticks(angles)\n",
    "    ax4.set_xticklabels(['Efficiency', 'Precision', 'Coverage'])\n",
    "    ax4.set_title('Defensive Performance Metrics')\n",
    "    ax4.legend(loc='upper right', bbox_to_anchor=(1.3, 1.1))\n",
    "    \n",
    "    # Overall title and adjustments\n",
    "    plt.suptitle('NFC West Defensive Movement Analysis Comparison', \n",
    "                size=16, y=1.02)\n",
    "    plt.tight_layout()\n",
    "    \n",
    "    # Save visualization\n",
    "    save_path = os.path.join(os.path.dirname(os.getcwd()), \n",
    "                            'final_visualizations', \n",
    "                            'nfc_west_comparison.png')\n",
    "    plt.savefig(save_path, dpi=300, bbox_inches='tight')\n",
    "    plt.close()\n",
    "    \n",
    "    return save_path\n",
    "\n",
    "# Create the comparison visualization\n",
    "print(\"Creating comparative analysis visualization...\")\n",
    "comparison_path = create_comparative_analysis()\n",
    "\n",
    "# Update documentation\n",
    "print(\"\\nUpdating documentation...\")\n",
    "manifest_path = os.path.join(os.path.dirname(os.getcwd()), \n",
    "                            'results', \n",
    "                            'final_visualization_manifest.md')\n",
    "\n",
    "with open(manifest_path, 'r') as f:\n",
    "    manifest_content = f.read()\n",
    "\n",
    "# Add new visualization to manifest\n",
    "new_entry = f\"\"\"\n",
    "### Comparative Analysis\n",
    "- **nfc_west_comparison.png**\n",
    "  - Comprehensive comparison of defensive movements across NFC West\n",
    "  - Highlights relative performance in efficiency, coverage, and pattern distribution\n",
    "  - Demonstrates {top_team}'s defensive movement characteristics in context\n",
    "\"\"\"\n",
    "\n",
    "with open(manifest_path, 'w') as f:\n",
    "    f.write(manifest_content + new_entry)\n",
    "\n",
    "print(f\"\\nComparative visualization saved to: {comparison_path}\")\n",
    "print(\"Manifest updated with new visualization\")\n",
    "\n",
    "# Show updated visualization count\n",
    "final_dir = os.path.join(os.path.dirname(os.getcwd()), 'final_visualizations')\n",
    "final_count = len(os.listdir(final_dir))\n",
    "print(f\"\\nTotal visualizations: {final_count}/9\")\n",
    "print(f\"Final memory usage: {get_memory_usage():.2f} MB\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ensemble",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}