jsb_data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "### IMPORTS ###\n",
    "import pandas as pd\n",
    "import os\n",
    "import yaml\n",
    "import numpy as np\n",
    "import json\n",
    "from itertools import combinations\n",
    "import pretty_midi\n",
    "import librosa\n",
    "import matplotlib.pyplot as plt\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "229 77 76\n",
      "305 77\n"
     ]
    }
   ],
   "source": [
    "# Load in original JSON file for chorale data\n",
    "json_path = './data/jsb-chorales-8th.json'\n",
    "with open(json_path, \"r\") as f:\n",
    "     chorale_data = json.loads(f.read())\n",
    "print(len(chorale_data['train']),len(chorale_data['test']),len(chorale_data['valid']))\n",
    "\n",
    "# combine train and validation sets since we don't need validation set\n",
    "chorale_data['train'] = chorale_data['train'] + chorale_data['valid']\n",
    "print(len(chorale_data['train']),len(chorale_data['test']))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "### SOME CHORD CONSTANTS\n",
    "notes_in_keys = {\n",
    "    #        C D E F G A B\n",
    "    \"C/a\":    [0,2,4,5,7,9,11], # C!\n",
    "    \"G/e\":    [0,2,4,6,7,9,11], #  F#\n",
    "    \"D/b\":    [1,2,4,6,7,9,11], # +C#\n",
    "    \"A/f#\":    [1,2,4,6,8,9,11], # +G#\n",
    "    \"E/c#\":    [1,3,4,6,8,9,11], # +D#\n",
    "    \"B-Cb/g#\": [1,3,4,6,8,10,11], # +A#\n",
    "    \"Gb-F#/eb\":[1,3,5,6,8,10,11], # +E#\n",
    "    \"Db-C#/bb\":[0,1,3,5,6,8,10], # +B#\n",
    "    \"Ab/f\":   [0,1,3,5,7,8,10], # +Db\n",
    "    \"Eb/c\":   [0,2,3,5,7,8,10], # +Ab\n",
    "    \"Bb/g\":   [0,2,3,5,7,9,10], # +Eb \n",
    "    \"F/d\":    [0,2,4,5,7,9,10]  # +Bb\n",
    "}\n",
    "\n",
    "maj_min_tonics = {\n",
    "    #        C D E F G A B\n",
    "    \"C/a\":  {'maj': [0,4,7], # CEG\n",
    "             'min': [0,4,9]}, # ACE\n",
    "    \"G/e\":  {'maj': [2,7,11], # GBD\n",
    "             'min': [4,7,11]},  # EGB\n",
    "    \"D/b\":  {'maj': [2,6,9], # DF#A \n",
    "             'min': [2,6,11]},  # +C#\n",
    "    \"A/f#\":   {'maj': [1,4,9], \n",
    "               'min': [1,6,9]}, # +G#\n",
    "    \"E/c#\":    {'maj': [4,8,11], # EG#B \n",
    "               'min': [1,4,8]}, # +D#\n",
    "    \"B-Cb/g#\": {'maj': [3,6,11], \n",
    "               'min': [3,8,11]}, # +A#\n",
    "    \"Gb-F#/eb\":{'maj': [1,6,10], \n",
    "               'min': [3,6,10]}, # +E#\n",
    "    \"Db-C#/bb\":{'maj': [1,5,8], # C# F G#\n",
    "               'min': [1,5,10]}, # +B#\n",
    "    \"Ab/f\":   {'maj': [0,3,8], # G# C D#\n",
    "               'min': [0,5,8]} , # +Db\n",
    "    \"Eb/c\":   {'maj': [3,7,10], # D#GA#\n",
    "               'min': [0,3,7]} , # +Ab\n",
    "    \"Bb/g\":   {'maj': [2,5,10], # D F Bb \n",
    "               'min': [2,7,10]},# +Eb \n",
    "    \"F/d\":    {'maj': [0,5,9], # FAC \n",
    "               'min': [2,5,9]}   # +Bb\n",
    "}\n",
    "\n",
    "notes_in_keys_minor = {\n",
    "    #        C D E F G A B\n",
    "    \"a\":    [0,2,4,5,7,8,9,11], # C!, raised 7th is G# (8)\n",
    "    \"e\":    [0,2,3,4,6,7,9,11], #  F# raised 7th is D# (3)\n",
    "    \"b\":    [1,2,4,6,7,9,10,11], # +C# raised 7th: A# 10\n",
    "    \"f#\":   [1,2,4,5,6,8,9,11], # +G# raised 7th: F 5\n",
    "    \"c#\":   [0,1,3,4,6,8,9,11], # +D#\n",
    "    \"g#\":   [1,3,4,5,6,8,10,11], # +A#\n",
    "    \"eb\":   [1,2,3,5,6,8,10,11], # +E#\n",
    "    \"bb\":   [0,1,3,5,6,8,9,10], # +B#\n",
    "    \"f\":    [0,1,3,4,5,7,8,10], # +Db\n",
    "    \"c\":    [0,2,3,5,7,8,10,11], # +Ab\n",
    "    \"g\":    [0,2,3,5,6,7,9,10], # +Eb \n",
    "    \"d\":    [0,1,2,4,5,7,9,10]  # +Bb\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "### CODE FOR ESTIMATING MODES AND KEYS\n",
    "def mode_estimate(chorale, key_estimate):\n",
    "    maj_tonic = set(maj_min_tonics[key_estimate]['maj'])\n",
    "    min_tonic = set(maj_min_tonics[key_estimate]['min'])\n",
    "    maj_tonic_count=0\n",
    "    min_tonic_count=0\n",
    "    for chord in chorale: \n",
    "        notes= set()\n",
    "        for pitch in chord: \n",
    "            note = pitch%12\n",
    "            if note not in notes: \n",
    "                notes.add(note)\n",
    "        if notes == maj_tonic:\n",
    "            maj_tonic_count += 1\n",
    "        if notes == min_tonic:\n",
    "            min_tonic_count += 1\n",
    "    if maj_tonic_count > min_tonic_count: \n",
    "        return \"maj\"\n",
    "    else: \n",
    "        return \"min\"\n",
    "        \n",
    "def key_estimate(chorale):\n",
    "    # Chorale is a list of lists of 4 pitches representing chords\n",
    "    note_counts = {0:0,1:0,2:0,3:0,4:0,5:0,6:0,7:0,8:0,9:0,10:0,11:0}\n",
    "    total_notes = len(chorale) * 4\n",
    "    for chord in chorale: # count # of occurences of every pitch in the chorale, then sort most to least frequent\n",
    "        for note in chord: \n",
    "            cur_note = note%12\n",
    "            note_counts[cur_note] +=1\n",
    "    sorted_notecounts = {k: v for k, v in sorted(note_counts.items(), key=lambda item: item[1], reverse=True)}\n",
    "\n",
    "    #print(sorted_notecounts)\n",
    "    # KEY_SUMS\n",
    "    max_proportion = 0\n",
    "    key_estimate = None\n",
    "    for key in notes_in_keys.keys():\n",
    "        key_sum = 0 \n",
    "        for note in notes_in_keys[key]:\n",
    "            key_sum += sorted_notecounts[note]\n",
    "        key_proportion = key_sum/total_notes\n",
    "        #print(key, key_sum/total_notes)\n",
    "        if key_proportion > max_proportion: \n",
    "            max_proportion = key_proportion\n",
    "            key_estimate = key\n",
    "    #print(max_proportion, key_estimate)\n",
    "    return key_estimate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: {'key': 'Bb/g', 'mode': 'maj'}, 1: {'key': 'A/f#', 'mode': 'maj'}, 2: {'key': 'G/e', 'mode': 'maj'}, 3: {'key': 'C/a', 'mode': 'maj'}, 4: {'key': 'C/a', 'mode': 'min'}, 5: {'key': 'D/b', 'mode': 'min'}, 6: {'key': 'Eb/c', 'mode': 'maj'}, 7: {'key': 'F/d', 'mode': 'min'}, 8: {'key': 'E/c#', 'mode': 'maj'}, 9: {'key': 'Bb/g', 'mode': 'min'}, 10: {'key': 'D/b', 'mode': 'maj'}, 11: {'key': 'Bb/g', 'mode': 'maj'}, 12: {'key': 'D/b', 'mode': 'min'}, 13: {'key': 'G/e', 'mode': 'maj'}, 14: {'key': 'C/a', 'mode': 'maj'}, 15: {'key': 'F/d', 'mode': 'maj'}, 16: {'key': 'F/d', 'mode': 'maj'}, 17: {'key': 'Eb/c', 'mode': 'maj'}, 18: {'key': 'Bb/g', 'mode': 'min'}, 19: {'key': 'Bb/g', 'mode': 'min'}, 20: {'key': 'A/f#', 'mode': 'maj'}, 21: {'key': 'G/e', 'mode': 'maj'}, 22: {'key': 'Eb/c', 'mode': 'maj'}, 23: {'key': 'D/b', 'mode': 'min'}, 24: {'key': 'C/a', 'mode': 'min'}, 25: {'key': 'D/b', 'mode': 'min'}, 26: {'key': 'C/a', 'mode': 'min'}, 27: {'key': 'A/f#', 'mode': 'maj'}, 28: {'key': 'Bb/g', 'mode': 'min'}, 29: {'key': 'G/e', 'mode': 'maj'}, 30: {'key': 'Bb/g', 'mode': 'maj'}, 31: {'key': 'F/d', 'mode': 'min'}, 32: {'key': 'C/a', 'mode': 'maj'}, 33: {'key': 'Bb/g', 'mode': 'min'}, 34: {'key': 'C/a', 'mode': 'min'}, 35: {'key': 'G/e', 'mode': 'maj'}, 36: {'key': 'C/a', 'mode': 'min'}, 37: {'key': 'Eb/c', 'mode': 'maj'}, 38: {'key': 'C/a', 'mode': 'min'}, 39: {'key': 'Bb/g', 'mode': 'min'}, 40: {'key': 'G/e', 'mode': 'maj'}, 41: {'key': 'Bb/g', 'mode': 'min'}, 42: {'key': 'G/e', 'mode': 'maj'}, 43: {'key': 'F/d', 'mode': 'maj'}, 44: {'key': 'G/e', 'mode': 'maj'}, 45: {'key': 'D/b', 'mode': 'maj'}, 46: {'key': 'G/e', 'mode': 'min'}, 47: {'key': 'F/d', 'mode': 'maj'}, 48: {'key': 'F/d', 'mode': 'maj'}, 49: {'key': 'C/a', 'mode': 'maj'}, 50: {'key': 'A/f#', 'mode': 'maj'}, 51: {'key': 'C/a', 'mode': 'maj'}, 52: {'key': 'D/b', 'mode': 'min'}, 53: {'key': 'C/a', 'mode': 'maj'}, 54: {'key': 'C/a', 'mode': 'maj'}, 55: {'key': 'Bb/g', 'mode': 'min'}, 56: {'key': 'C/a', 'mode': 'min'}, 57: {'key': 'E/c#', 'mode': 'maj'}, 58: {'key': 'Bb/g', 'mode': 'maj'}, 59: {'key': 'Bb/g', 'mode': 'min'}, 60: {'key': 'E/c#', 'mode': 'maj'}, 61: {'key': 'F/d', 'mode': 'maj'}, 62: {'key': 'A/f#', 'mode': 'min'}, 63: {'key': 'D/b', 'mode': 'min'}, 64: {'key': 'G/e', 'mode': 'maj'}, 65: {'key': 'Bb/g', 'mode': 'min'}, 66: {'key': 'Bb/g', 'mode': 'maj'}, 67: {'key': 'F/d', 'mode': 'min'}, 68: {'key': 'D/b', 'mode': 'maj'}, 69: {'key': 'Ab/f', 'mode': 'maj'}, 70: {'key': 'A/f#', 'mode': 'min'}, 71: {'key': 'D/b', 'mode': 'maj'}, 72: {'key': 'C/a', 'mode': 'min'}, 73: {'key': 'Bb/g', 'mode': 'min'}, 74: {'key': 'Bb/g', 'mode': 'min'}, 75: {'key': 'G/e', 'mode': 'maj'}, 76: {'key': 'D/b', 'mode': 'maj'}, 77: {'key': 'E/c#', 'mode': 'maj'}, 78: {'key': 'G/e', 'mode': 'maj'}, 79: {'key': 'A/f#', 'mode': 'maj'}, 80: {'key': 'G/e', 'mode': 'maj'}, 81: {'key': 'E/c#', 'mode': 'maj'}, 82: {'key': 'G/e', 'mode': 'maj'}, 83: {'key': 'F/d', 'mode': 'maj'}, 84: {'key': 'F/d', 'mode': 'maj'}, 85: {'key': 'C/a', 'mode': 'min'}, 86: {'key': 'G/e', 'mode': 'min'}, 87: {'key': 'C/a', 'mode': 'maj'}, 88: {'key': 'C/a', 'mode': 'maj'}, 89: {'key': 'C/a', 'mode': 'min'}, 90: {'key': 'G/e', 'mode': 'maj'}, 91: {'key': 'C/a', 'mode': 'maj'}, 92: {'key': 'C/a', 'mode': 'min'}, 93: {'key': 'Bb/g', 'mode': 'min'}, 94: {'key': 'F/d', 'mode': 'maj'}, 95: {'key': 'G/e', 'mode': 'maj'}, 96: {'key': 'E/c#', 'mode': 'maj'}, 97: {'key': 'C/a', 'mode': 'min'}, 98: {'key': 'A/f#', 'mode': 'maj'}, 99: {'key': 'G/e', 'mode': 'maj'}, 100: {'key': 'D/b', 'mode': 'min'}, 101: {'key': 'G/e', 'mode': 'maj'}, 102: {'key': 'Eb/c', 'mode': 'maj'}, 103: {'key': 'G/e', 'mode': 'maj'}, 104: {'key': 'D/b', 'mode': 'maj'}, 105: {'key': 'G/e', 'mode': 'maj'}, 106: {'key': 'G/e', 'mode': 'maj'}, 107: {'key': 'C/a', 'mode': 'min'}, 108: {'key': 'G/e', 'mode': 'min'}, 109: {'key': 'Bb/g', 'mode': 'maj'}, 110: {'key': 'Bb/g', 'mode': 'maj'}, 111: {'key': 'A/f#', 'mode': 'maj'}, 112: {'key': 'F/d', 'mode': 'maj'}, 113: {'key': 'D/b', 'mode': 'maj'}, 114: {'key': 'C/a', 'mode': 'min'}, 115: {'key': 'C/a', 'mode': 'min'}, 116: {'key': 'C/a', 'mode': 'maj'}, 117: {'key': 'F/d', 'mode': 'maj'}, 118: {'key': 'C/a', 'mode': 'min'}, 119: {'key': 'C/a', 'mode': 'min'}, 120: {'key': 'Eb/c', 'mode': 'min'}, 121: {'key': 'C/a', 'mode': 'min'}, 122: {'key': 'C/a', 'mode': 'min'}, 123: {'key': 'D/b', 'mode': 'maj'}, 124: {'key': 'C/a', 'mode': 'min'}, 125: {'key': 'F/d', 'mode': 'maj'}, 126: {'key': 'E/c#', 'mode': 'maj'}, 127: {'key': 'F/d', 'mode': 'min'}, 128: {'key': 'Bb/g', 'mode': 'maj'}, 129: {'key': 'G/e', 'mode': 'maj'}, 130: {'key': 'Bb/g', 'mode': 'maj'}, 131: {'key': 'Eb/c', 'mode': 'min'}, 132: {'key': 'G/e', 'mode': 'maj'}, 133: {'key': 'D/b', 'mode': 'maj'}, 134: {'key': 'C/a', 'mode': 'min'}, 135: {'key': 'G/e', 'mode': 'maj'}, 136: {'key': 'D/b', 'mode': 'maj'}, 137: {'key': 'F/d', 'mode': 'maj'}, 138: {'key': 'F/d', 'mode': 'maj'}, 139: {'key': 'D/b', 'mode': 'maj'}, 140: {'key': 'C/a', 'mode': 'maj'}, 141: {'key': 'G/e', 'mode': 'maj'}, 142: {'key': 'Eb/c', 'mode': 'min'}, 143: {'key': 'E/c#', 'mode': 'maj'}, 144: {'key': 'D/b', 'mode': 'maj'}, 145: {'key': 'F/d', 'mode': 'maj'}, 146: {'key': 'Bb/g', 'mode': 'min'}, 147: {'key': 'D/b', 'mode': 'maj'}, 148: {'key': 'C/a', 'mode': 'min'}, 149: {'key': 'Bb/g', 'mode': 'min'}, 150: {'key': 'F/d', 'mode': 'min'}, 151: {'key': 'D/b', 'mode': 'min'}, 152: {'key': 'F/d', 'mode': 'maj'}, 153: {'key': 'C/a', 'mode': 'maj'}, 154: {'key': 'C/a', 'mode': 'maj'}, 155: {'key': 'C/a', 'mode': 'min'}, 156: {'key': 'C/a', 'mode': 'maj'}, 157: {'key': 'A/f#', 'mode': 'maj'}, 158: {'key': 'C/a', 'mode': 'min'}, 159: {'key': 'Bb/g', 'mode': 'min'}, 160: {'key': 'D/b', 'mode': 'min'}, 161: {'key': 'C/a', 'mode': 'maj'}, 162: {'key': 'Bb/g', 'mode': 'min'}, 163: {'key': 'D/b', 'mode': 'maj'}, 164: {'key': 'Bb/g', 'mode': 'min'}, 165: {'key': 'C/a', 'mode': 'min'}, 166: {'key': 'D/b', 'mode': 'maj'}, 167: {'key': 'C/a', 'mode': 'min'}, 168: {'key': 'Bb/g', 'mode': 'maj'}, 169: {'key': 'D/b', 'mode': 'maj'}, 170: {'key': 'A/f#', 'mode': 'maj'}, 171: {'key': 'Bb/g', 'mode': 'maj'}, 172: {'key': 'C/a', 'mode': 'min'}, 173: {'key': 'G/e', 'mode': 'min'}, 174: {'key': 'A/f#', 'mode': 'maj'}, 175: {'key': 'E/c#', 'mode': 'maj'}, 176: {'key': 'G/e', 'mode': 'maj'}, 177: {'key': 'A/f#', 'mode': 'maj'}, 178: {'key': 'F/d', 'mode': 'maj'}, 179: {'key': 'G/e', 'mode': 'maj'}, 180: {'key': 'C/a', 'mode': 'maj'}, 181: {'key': 'C/a', 'mode': 'min'}, 182: {'key': 'C/a', 'mode': 'min'}, 183: {'key': 'F/d', 'mode': 'maj'}, 184: {'key': 'G/e', 'mode': 'min'}, 185: {'key': 'C/a', 'mode': 'maj'}, 186: {'key': 'D/b', 'mode': 'min'}, 187: {'key': 'G/e', 'mode': 'maj'}, 188: {'key': 'Bb/g', 'mode': 'min'}, 189: {'key': 'A/f#', 'mode': 'maj'}, 190: {'key': 'F/d', 'mode': 'maj'}, 191: {'key': 'G/e', 'mode': 'maj'}, 192: {'key': 'A/f#', 'mode': 'maj'}, 193: {'key': 'G/e', 'mode': 'maj'}, 194: {'key': 'C/a', 'mode': 'maj'}, 195: {'key': 'C/a', 'mode': 'min'}, 196: {'key': 'Bb/g', 'mode': 'maj'}, 197: {'key': 'C/a', 'mode': 'min'}, 198: {'key': 'A/f#', 'mode': 'min'}, 199: {'key': 'A/f#', 'mode': 'maj'}, 200: {'key': 'C/a', 'mode': 'min'}, 201: {'key': 'G/e', 'mode': 'min'}, 202: {'key': 'Eb/c', 'mode': 'maj'}, 203: {'key': 'Bb/g', 'mode': 'maj'}, 204: {'key': 'A/f#', 'mode': 'maj'}, 205: {'key': 'G/e', 'mode': 'maj'}, 206: {'key': 'D/b', 'mode': 'min'}, 207: {'key': 'G/e', 'mode': 'maj'}, 208: {'key': 'Bb/g', 'mode': 'min'}, 209: {'key': 'C/a', 'mode': 'maj'}, 210: {'key': 'A/f#', 'mode': 'maj'}, 211: {'key': 'F/d', 'mode': 'min'}, 212: {'key': 'F/d', 'mode': 'maj'}, 213: {'key': 'F/d', 'mode': 'maj'}, 214: {'key': 'A/f#', 'mode': 'maj'}, 215: {'key': 'A/f#', 'mode': 'maj'}, 216: {'key': 'C/a', 'mode': 'maj'}, 217: {'key': 'D/b', 'mode': 'min'}, 218: {'key': 'D/b', 'mode': 'maj'}, 219: {'key': 'F/d', 'mode': 'min'}, 220: {'key': 'Bb/g', 'mode': 'min'}, 221: {'key': 'A/f#', 'mode': 'min'}, 222: {'key': 'D/b', 'mode': 'maj'}, 223: {'key': 'D/b', 'mode': 'maj'}, 224: {'key': 'G/e', 'mode': 'maj'}, 225: {'key': 'Bb/g', 'mode': 'maj'}, 226: {'key': 'Bb/g', 'mode': 'min'}, 227: {'key': 'C/a', 'mode': 'maj'}, 228: {'key': 'A/f#', 'mode': 'maj'}, 229: {'key': 'C/a', 'mode': 'maj'}, 230: {'key': 'C/a', 'mode': 'maj'}, 231: {'key': 'C/a', 'mode': 'maj'}, 232: {'key': 'Eb/c', 'mode': 'maj'}, 233: {'key': 'F/d', 'mode': 'maj'}, 234: {'key': 'D/b', 'mode': 'min'}, 235: {'key': 'Bb/g', 'mode': 'maj'}, 236: {'key': 'D/b', 'mode': 'maj'}, 237: {'key': 'C/a', 'mode': 'min'}, 238: {'key': 'C/a', 'mode': 'min'}, 239: {'key': 'A/f#', 'mode': 'maj'}, 240: {'key': 'Bb/g', 'mode': 'min'}, 241: {'key': 'C/a', 'mode': 'min'}, 242: {'key': 'D/b', 'mode': 'min'}, 243: {'key': 'D/b', 'mode': 'min'}, 244: {'key': 'G/e', 'mode': 'maj'}, 245: {'key': 'D/b', 'mode': 'min'}, 246: {'key': 'C/a', 'mode': 'min'}, 247: {'key': 'A/f#', 'mode': 'min'}, 248: {'key': 'G/e', 'mode': 'min'}, 249: {'key': 'Bb/g', 'mode': 'maj'}, 250: {'key': 'G/e', 'mode': 'maj'}, 251: {'key': 'A/f#', 'mode': 'maj'}, 252: {'key': 'G/e', 'mode': 'maj'}, 253: {'key': 'Bb/g', 'mode': 'maj'}, 254: {'key': 'G/e', 'mode': 'maj'}, 255: {'key': 'Bb/g', 'mode': 'min'}, 256: {'key': 'D/b', 'mode': 'min'}, 257: {'key': 'Bb/g', 'mode': 'maj'}, 258: {'key': 'Bb/g', 'mode': 'min'}, 259: {'key': 'F/d', 'mode': 'min'}, 260: {'key': 'C/a', 'mode': 'min'}, 261: {'key': 'A/f#', 'mode': 'maj'}, 262: {'key': 'G/e', 'mode': 'maj'}, 263: {'key': 'D/b', 'mode': 'min'}, 264: {'key': 'G/e', 'mode': 'maj'}, 265: {'key': 'D/b', 'mode': 'maj'}, 266: {'key': 'C/a', 'mode': 'min'}, 267: {'key': 'G/e', 'mode': 'maj'}, 268: {'key': 'Eb/c', 'mode': 'maj'}, 269: {'key': 'C/a', 'mode': 'maj'}, 270: {'key': 'F/d', 'mode': 'maj'}, 271: {'key': 'A/f#', 'mode': 'maj'}, 272: {'key': 'D/b', 'mode': 'maj'}, 273: {'key': 'D/b', 'mode': 'min'}, 274: {'key': 'Eb/c', 'mode': 'maj'}, 275: {'key': 'Bb/g', 'mode': 'min'}, 276: {'key': 'C/a', 'mode': 'maj'}, 277: {'key': 'A/f#', 'mode': 'maj'}, 278: {'key': 'D/b', 'mode': 'maj'}, 279: {'key': 'E/c#', 'mode': 'maj'}, 280: {'key': 'Db-C#/bb', 'mode': 'min'}, 281: {'key': 'Bb/g', 'mode': 'maj'}, 282: {'key': 'G/e', 'mode': 'maj'}, 283: {'key': 'G/e', 'mode': 'min'}, 284: {'key': 'C/a', 'mode': 'min'}, 285: {'key': 'D/b', 'mode': 'min'}, 286: {'key': 'G/e', 'mode': 'maj'}, 287: {'key': 'C/a', 'mode': 'min'}, 288: {'key': 'F/d', 'mode': 'maj'}, 289: {'key': 'D/b', 'mode': 'min'}, 290: {'key': 'G/e', 'mode': 'maj'}, 291: {'key': 'G/e', 'mode': 'min'}, 292: {'key': 'C/a', 'mode': 'min'}, 293: {'key': 'G/e', 'mode': 'maj'}, 294: {'key': 'D/b', 'mode': 'maj'}, 295: {'key': 'G/e', 'mode': 'min'}, 296: {'key': 'G/e', 'mode': 'maj'}, 297: {'key': 'Bb/g', 'mode': 'maj'}, 298: {'key': 'G/e', 'mode': 'maj'}, 299: {'key': 'F/d', 'mode': 'maj'}, 300: {'key': 'C/a', 'mode': 'maj'}, 301: {'key': 'Bb/g', 'mode': 'maj'}, 302: {'key': 'D/b', 'mode': 'maj'}, 303: {'key': 'A/f#', 'mode': 'maj'}, 304: {'key': 'G/e', 'mode': 'maj'}}\n"
     ]
    }
   ],
   "source": [
    "train_keys = {}\n",
    "for i, chorale in enumerate(chorale_data['train']):\n",
    "    key = key_estimate(chorale)\n",
    "    train_keys[int(i)]={}\n",
    "    train_keys[int(i)][\"key\"] = key\n",
    "    train_keys[int(i)][\"mode\"] = mode_estimate(chorale, key)\n",
    "\n",
    "test_keys = {}\n",
    "for i, chorale in enumerate(chorale_data['test']):\n",
    "    key = key_estimate(chorale)\n",
    "    test_keys[i]={}\n",
    "    test_keys[i][\"key\"] = key\n",
    "    test_keys[i][\"mode\"] = mode_estimate(chorale, key)\n",
    "\n",
    "print(train_keys)\n",
    "\n",
    "keys_and_modes = {\n",
    "    \"train\": train_keys,\n",
    "    \"test\": test_keys,\n",
    "}\n",
    "\n",
    "# save dictionary with key and mode estimates for each chorale\n",
    "with open('./data/jsb_chorales_keys_modes.yaml', 'w') as outfile:\n",
    "    yaml.dump(keys_and_modes, outfile, default_flow_style=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[55, 84] [50, 79] [43, 74] [31, 67]\n",
      "[55, 84] [50, 77] [45, 71] [31, 65]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([55, 84], [50, 77], [45, 71], [31, 65])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we want to shift all chorales into the key of C.\n",
    "key_shift_amount = { # SUBTRACT!!!! move the least amount possible, so sometimes add, sometimes subtract\n",
    "    \"C/a\":      0, # C!\n",
    "    \"G/e\":      5, #  F# \n",
    "    \"D/b\":      -2, # +C#\n",
    "    \"A/f#\":     3, # +G#\n",
    "    \"E/c#\":     -4, # +D#\n",
    "    \"B-Cb/g#\":  1, # +A#\n",
    "    \"Gb-F#/eb\": -6, # +E#\n",
    "    \"Db-C#/bb\": -1,#-1, # +B# \n",
    "    \"Ab/f\":     4, # +Db\n",
    "    \"Eb/c\":     -3, # +Ab\n",
    "    \"Bb/g\":     2, # +Eb \n",
    "    \"F/d\":      -5   # +Bb\n",
    "}\n",
    "\n",
    "def percentage_in_c(chorale):\n",
    "    in_c = 0\n",
    "    notes_in_c = [0,2,4,5,7,9,11]\n",
    "    for chord in chorale:\n",
    "        modded_chord = [n%12 for n in chord] \n",
    "        if set(modded_chord).issubset(set(notes_in_c)):\n",
    "            in_c += 1\n",
    "    return in_c / len(chorale)\n",
    "\n",
    "def shift_to_c(chorale_data, ttv):\n",
    "    shifted_chorales = []\n",
    "    for i, chorale in enumerate(chorale_data[ttv]):\n",
    "        key = keys_and_modes[ttv][i]['key']\n",
    "        mode = keys_and_modes[ttv][i]['mode']\n",
    "        shift_amount = key_shift_amount[key]\n",
    "        new_chorale = [] \n",
    "        for chord in chorale: \n",
    "            new_chord = [] \n",
    "            for note in chord: \n",
    "                new_chord.append(int(note + shift_amount))\n",
    "            new_chord.sort() # want it to go BTAS\n",
    "            new_chorale.append(new_chord)\n",
    "        shifted_chorales.append(new_chorale)\n",
    "    return shifted_chorales\n",
    "\n",
    "def determine_ranges(chorale_data): \n",
    "    # determine new part ranges\n",
    "    bass_mm = [128,0] # min, then max\n",
    "    ten_mm = [128,0]\n",
    "    alt_mm = [128,0]\n",
    "    sop_mm = [128,0]\n",
    "    for i, chorale in enumerate(chorale_data): # enumerate(chorales_in_c['train']):\n",
    "        for chord in chorale:\n",
    "            if len(chord) == 4:\n",
    "                if chord[0] > bass_mm[1]:\n",
    "                    bass_mm[1] = chord[0]\n",
    "                if chord[0] < bass_mm[0]:\n",
    "                    bass_mm[0] = chord[0]\n",
    "                if chord[1] > ten_mm[1]:\n",
    "                    ten_mm[1] = chord[1]\n",
    "                if chord[1] < ten_mm[0]:\n",
    "                    ten_mm[0] = chord[1]\n",
    "                if chord[2] > alt_mm[1]:\n",
    "                    alt_mm[1] = chord[2]\n",
    "                if chord[2] < alt_mm[0]:\n",
    "                    alt_mm[0] = chord[2]\n",
    "                if chord[3] > sop_mm[1]:\n",
    "                    sop_mm[1] = chord[3]\n",
    "                if chord[3] < sop_mm[0]:\n",
    "                    sop_mm[0] = chord[3] \n",
    "    print(sop_mm, alt_mm, ten_mm, bass_mm)\n",
    "    return sop_mm, alt_mm, ten_mm, bass_mm\n",
    "\n",
    "train_in_c = shift_to_c(chorale_data, 'train')\n",
    "test_in_c = shift_to_c(chorale_data, 'test')\n",
    "\n",
    "determine_ranges(train_in_c)\n",
    "determine_ranges(test_in_c)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "chorales_in_c = {\n",
    "    \"train\": train_in_c,\n",
    "    \"test\": test_in_c\n",
    "}\n",
    "# save dictionary with key and mode estimates for each chorale\n",
    "with open('./data/jsb_chorales_in_c.yaml', 'w') as outfile: # STILL HAS MAJOR AND MINOR\n",
    "    yaml.dump(chorales_in_c, outfile, default_flow_style=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "305 77\n",
      "190 50\n"
     ]
    }
   ],
   "source": [
    "def get_voicings_durations(chorale_data, ttv): \n",
    "    all_voicings = []\n",
    "    all_durations = []\n",
    "    maj_count = 0\n",
    "    for i, chorale in enumerate(chorale_data[ttv]):\n",
    "        if keys_and_modes[ttv][i]['mode'] == 'maj': # ONLY MAJOR!!\n",
    "            maj_count += 1\n",
    "            cur_voicing = [] \n",
    "            cur_durations = []\n",
    "            cur_dur = 0\n",
    "            cur_voc = None\n",
    "            for j,chord in enumerate(chorale):\n",
    "                if chord is not None and len(chord) == 4:\n",
    "                    if j == 0:\n",
    "                        cur_voc = chord\n",
    "                        cur_dur = 1\n",
    "                    else: \n",
    "                        if j == len(chorale) -1: \n",
    "                            cur_voicing.append(cur_voc) # append old variables\n",
    "                            if chord != cur_voc:\n",
    "                                cur_durations.append(cur_dur)\n",
    "                                cur_voicing.append(chord)\n",
    "                                cur_durations.append(1)\n",
    "                            else:\n",
    "                                cur_durations.append(cur_dur + 1)\n",
    "                            break\n",
    "                        if chord != cur_voc:\n",
    "                            cur_voicing.append(cur_voc) # append old variables\n",
    "                            cur_durations.append(cur_dur)\n",
    "                            cur_voc = chord  # reset current variables\n",
    "                            cur_dur = 1\n",
    "                        else: # no chord change\n",
    "                            cur_dur += 1\n",
    "            assert len(cur_voicing) == len(cur_durations), \"LIST LENGTH MISMATCH\"\n",
    "            all_voicings.append(cur_voicing) \n",
    "            all_durations.append(cur_durations) \n",
    "\n",
    "    return all_voicings, all_durations      \n",
    "\n",
    "\n",
    "print(len(chorales_in_c['train']), len(chorales_in_c['test']))\n",
    "training_voc, training_dur = get_voicings_durations(chorales_in_c, 'train')\n",
    "testing_voc, testing_dur = get_voicings_durations(chorales_in_c, 'test')\n",
    "print(len(training_voc), len(testing_voc)) # 190 major training examples, 50 major testing examples\n",
    "\n",
    "voicing_duration_dict = {\n",
    "    'train': {\n",
    "        'voicings': training_voc,\n",
    "        'durations': training_dur\n",
    "    },\n",
    "    'test': {\n",
    "        'voicings': testing_voc,\n",
    "        'durations': testing_dur\n",
    "    }\n",
    "}\n",
    "\n",
    "with open('./data/jsb_maj_voicings_durations.yaml', 'w') as outfile:\n",
    "    yaml.dump(voicing_duration_dict, outfile, default_flow_style=False) # STILL ON A PER-CHORALE BASIS! ONLY MAJOR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOTAL CHUNKS: 1239\n",
      "TOTAL CHUNKS: 375\n",
      "1239 1239 1239 1239\n",
      "375 375 375 375\n",
      "[1, 5, 10, 1, 5, 6, -1]\n",
      "[[60], [67], [65], [64], [62], [60], [-1]]\n",
      "[2, 1, 1, 1, 1, 1]\n"
     ]
    }
   ],
   "source": [
    "### TIME TO CHUNK!\n",
    "# NOW GET DATA CHUNKS:\n",
    "notes_in_chords = {\n",
    "    1: [0,4,7], # C\n",
    "    2: [2,5,9], # d\n",
    "    3: [4,7,11], # e\n",
    "    4: [5,9,0], # F\n",
    "    5: [7,11,2], # G\n",
    "    6: [9,0,4], # a\n",
    "    7: [11,2,5], # b dim\n",
    "    8: [2,5,9,0], # 2 7th\n",
    "    9: [5,9,0,4], # 4 7th \n",
    "    10: [7,11,2,5], # 5 7th \n",
    "    11: [11,2,5,9] # 7 7th # might add additional chords!!\n",
    "}\n",
    "\n",
    "def get_chord_idx(chord):\n",
    "    notes = set([n%12 for n in chord])\n",
    "    chord_idx = -1\n",
    "    for idx in notes_in_chords.keys():\n",
    "        if notes == set(notes_in_chords[idx]):\n",
    "            return idx\n",
    "    return chord_idx\n",
    "\n",
    "def chunk(voicings, durations):\n",
    "    num_chunks = 0\n",
    "    voc_chunks = [] \n",
    "    chord_chunks = [] \n",
    "    melody_chunks = [] \n",
    "    dur_chunks = []\n",
    "    for voicing_list, dur_list in zip(voicings, durations): \n",
    "        cur_chunk = [] \n",
    "        cur_chord_chunk = [] \n",
    "        cur_mel_chunk = [] \n",
    "        cur_dur_chunk = []\n",
    "        invalid_chords = 0\n",
    "        for chord, dur in zip(voicing_list, dur_list):\n",
    "            if chord is not None and len(chord) == 4:\n",
    "                idx = get_chord_idx(chord)\n",
    "                if idx == -1: # if we don't recognize the chord, end chunk and start a new one\n",
    "                    invalid_chords += 1\n",
    "                    if len(cur_chunk) > 2:\n",
    "                        num_chunks += 1\n",
    "                        voc_chunks.append(cur_chunk) # don't need to append -1s because wont be running alg on them. \n",
    "                        cur_chord_chunk.append(-1)\n",
    "                        chord_chunks.append(cur_chord_chunk)\n",
    "                        cur_mel_chunk.append(-1)\n",
    "                        melody_chunks.append(cur_mel_chunk)\n",
    "                        dur_chunks.append(cur_dur_chunk)\n",
    "                    cur_chunk = []\n",
    "                    cur_chord_chunk = [] \n",
    "                    cur_mel_chunk = []\n",
    "                    cur_dur_chunk = []\n",
    "                else: \n",
    "                    cur_chunk.append(chord)\n",
    "                    cur_chord_chunk.append(idx)\n",
    "                    cur_mel_chunk.append(chord[-1])\n",
    "                    cur_dur_chunk.append(dur)\n",
    "        if len(cur_chunk) >2:\n",
    "            num_chunks += 1\n",
    "            voc_chunks.append(cur_chunk)\n",
    "            cur_chord_chunk.append(-1)\n",
    "            chord_chunks.append(cur_chord_chunk)\n",
    "            cur_mel_chunk.append(-1)\n",
    "            melody_chunks.append(cur_mel_chunk)\n",
    "            dur_chunks.append(cur_dur_chunk)\n",
    "    print(\"TOTAL CHUNKS:\", num_chunks)\n",
    "    return voc_chunks, chord_chunks, melody_chunks, dur_chunks\n",
    "\n",
    "training_voc_chunks, training_chord_chunks, training_mel_chunks, training_dur_chunks = chunk(training_voc, training_dur)\n",
    "testing_voc_chunks, testing_chord_chunks, testing_mel_chunks, testing_dur_chunks = chunk(testing_voc, testing_dur)\n",
    "### MELODIES HAVE TO BE LISTS OF LISTS \n",
    "testing_mel_list_chunks = [[[note] for note in mel] for mel in testing_mel_chunks]\n",
    "training_mel_list_chunks = [[[note] for note in mel] for mel in training_mel_chunks]\n",
    "\n",
    "print(len(training_voc_chunks),len(training_chord_chunks),len(training_mel_chunks), len(training_dur_chunks))\n",
    "print(len(testing_voc_chunks), len(testing_chord_chunks), len(testing_mel_chunks), len(testing_dur_chunks))\n",
    "print(testing_chord_chunks[0])\n",
    "print(testing_mel_list_chunks[0])\n",
    "print(testing_dur_chunks[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "### SAVE MY DATASETS!\n",
    "orig_voicings = {\n",
    "    'train': training_voc_chunks,\n",
    "    'test': testing_voc_chunks\n",
    "}\n",
    "\n",
    "mel_data = {\n",
    "    'train': training_mel_list_chunks,\n",
    "    'test': testing_mel_list_chunks\n",
    "}\n",
    "\n",
    "chord_data = {\n",
    "    'train': training_chord_chunks,\n",
    "    'test': testing_chord_chunks\n",
    "}\n",
    "\n",
    "dur_data = {\n",
    "    'train': training_dur_chunks,\n",
    "    'test': testing_dur_chunks\n",
    "}\n",
    "\n",
    "with open('./data/jsb_maj_orig_voicings.yaml', 'w') as outfile:\n",
    "    yaml.dump(orig_voicings, outfile, default_flow_style=False)\n",
    "\n",
    "with open('./data/jsb_maj_chord_progs.yaml', 'w') as outfile:\n",
    "    yaml.dump(chord_data, outfile, default_flow_style=False)\n",
    "              \n",
    "with open('./data/jsb_maj_melodies.yaml', 'w') as outfile:\n",
    "    yaml.dump(mel_data, outfile, default_flow_style=False)\n",
    "\n",
    "with open('./data/jsb_maj_durations.yaml', 'w') as outfile:\n",
    "    yaml.dump(dur_data, outfile, default_flow_style=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3\n",
      "0.296\n",
      "0.292\n",
      "0.288\n",
      "0.28400000000000003\n",
      "0.28\n",
      "0.276\n",
      "0.272\n",
      "0.268\n",
      "0.264\n",
      "0.26\n",
      "0.256\n",
      "0.252\n",
      "0.248\n",
      "0.244\n",
      "0.24\n",
      "0.23600000000000002\n",
      "0.232\n",
      "0.228\n",
      "0.22399999999999998\n",
      "0.21999999999999997\n",
      "0.21599999999999997\n",
      "0.21200000000000002\n",
      "0.20800000000000002\n",
      "0.20400000000000001\n",
      "0.2\n",
      "0.196\n",
      "0.192\n",
      "0.188\n",
      "0.184\n",
      "0.18\n",
      "0.176\n",
      "0.172\n",
      "0.168\n",
      "0.164\n",
      "0.16\n",
      "0.156\n",
      "0.152\n",
      "0.148\n",
      "0.14400000000000002\n",
      "0.14\n",
      "0.136\n",
      "0.132\n",
      "0.128\n",
      "0.124\n",
      "0.12000000000000001\n",
      "0.116\n",
      "0.112\n",
      "0.10800000000000001\n",
      "0.10400000000000001\n",
      "0.27\n",
      "0.24300000000000002\n",
      "0.21870000000000003\n",
      "0.19683000000000003\n",
      "0.17714700000000003\n",
      "0.15943230000000003\n",
      "0.14348907000000002\n",
      "0.12914016300000003\n",
      "0.11622614670000003\n",
      "0.10460353203000003\n",
      "0.09414317882700003\n",
      "0.08472886094430003\n",
      "0.07625597484987003\n",
      "0.06863037736488302\n",
      "0.061767339628394716\n",
      "0.055590605665555244\n",
      "0.05003154509899972\n",
      "0.04502839058909975\n",
      "0.040525551530189774\n",
      "0.036472996377170795\n",
      "0.032825696739453715\n",
      "0.029543127065508344\n",
      "0.026588814358957512\n",
      "0.02392993292306176\n",
      "0.021536939630755585\n",
      "0.019383245667680026\n",
      "0.017444921100912024\n",
      "0.015700428990820824\n",
      "0.014130386091738742\n",
      "0.012717347482564867\n",
      "0.01144561273430838\n",
      "0.010301051460877543\n",
      "0.009270946314789788\n",
      "0.00834385168331081\n",
      "0.007509466514979729\n",
      "0.006758519863481757\n",
      "0.006082667877133581\n",
      "0.005474401089420223\n",
      "0.004926960980478201\n",
      "0.004434264882430382\n",
      "0.003990838394187343\n",
      "0.003591754554768609\n",
      "0.003232579099291748\n",
      "0.0029093211893625732\n",
      "0.002618389070426316\n",
      "0.0023565501633836844\n",
      "0.002120895147045316\n",
      "0.0019088056323407843\n",
      "0.001717925069106706\n",
      "0.0015461325621960354\n"
     ]
    }
   ],
   "source": [
    "start = 0.3\n",
    "end = 0.1\n",
    "for i in range(50):\n",
    "    print((start-end)*((50 - i)/50) + end, )\n",
    "\n",
    "for i in range(50):\n",
    "    start = start * 0.9 \n",
    "    print(start)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "midigen2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}