Card similarity.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import codecs\n",
    "import collections\n",
    "import gzip\n",
    "import itertools\n",
    "import json\n",
    "import re\n",
    "from operator import itemgetter\n",
    "\n",
    "import nltk.stem, nltk.corpus\n",
    "from gensim import corpora, models, similarities"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# curl -O https://mtgjson.com/json/AllCards.json.gz\n",
    "fname = 'AllCards.json.gz'\n",
    "cards = json.load(gzip.open(fname, 'rt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colorIdentity': ['G'],\n",
       " 'colors': ['G'],\n",
       " 'convertedManaCost': 1.0,\n",
       " 'foreignData': [{'language': 'German',\n",
       "   'name': 'Mooshund',\n",
       "   'text': 'Immer wenn der Mooshund das Ziel eines Zauberspruchs oder einer Fähigkeit ist, die ein Gegner kontrolliert, lege eine +1/+1-Marke auf den Mooshund.',\n",
       "   'type': 'Kreatur — Hund'},\n",
       "  {'language': 'Spanish', 'name': 'Perro de musgo'},\n",
       "  {'language': 'French', 'name': 'Chien de mousse'},\n",
       "  {'language': 'Italian', 'name': 'Cane di Muschio'},\n",
       "  {'language': 'Japanese', 'name': '苔犬'},\n",
       "  {'language': 'Portuguese (Brazil)', 'name': 'Cão de Musgo'}],\n",
       " 'layout': 'normal',\n",
       " 'legalities': {'commander': 'Legal',\n",
       "  'duel': 'Legal',\n",
       "  'legacy': 'Legal',\n",
       "  'pauper': 'Legal',\n",
       "  'penny': 'Legal',\n",
       "  'vintage': 'Legal'},\n",
       " 'manaCost': '{G}',\n",
       " 'mtgstocksId': 8192,\n",
       " 'name': 'Mossdog',\n",
       " 'power': '1',\n",
       " 'printings': ['NEM'],\n",
       " 'purchaseUrls': {'cardmarket': 'https://mtgjson.com/links/cc00eafc98e25a19',\n",
       "  'mtgstocks': 'https://mtgjson.com/links/2caa0df46f99b875',\n",
       "  'tcgplayer': 'https://mtgjson.com/links/ac2626141862f266'},\n",
       " 'rulings': [],\n",
       " 'scryfallOracleId': '95cc3652-b78b-473b-84cf-0dd3a7006219',\n",
       " 'subtypes': ['Plant', 'Hound'],\n",
       " 'supertypes': [],\n",
       " 'text': 'Whenever Mossdog becomes the target of a spell or ability an opponent controls, put a +1/+1 counter on Mossdog.',\n",
       " 'toughness': '1',\n",
       " 'type': 'Creature — Plant Hound',\n",
       " 'types': ['Creature'],\n",
       " 'uuid': '3e3d5801-0376-5d35-8171-210bc6a5e678'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cards['Mossdog']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "card_names = list(cards.keys())\n",
    "with open('card_names.txt', 'w') as f:\n",
    "    f.writelines(t + '\\n' for t in card_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/clayton_davis/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "nltk.download('stopwords')\n",
    "stopwords = set(nltk.corpus.stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = nltk.stem.snowball.SnowballStemmer('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(card):\n",
    "    text = ' '.join([card.get('text', '')]\n",
    "                   # + card.get('types', [])\n",
    "                   + card.get('subtypes', [])\n",
    "                    )\n",
    "    text = text.lower()\n",
    "    ## Replace card name with ~\n",
    "    text = text.replace(card['name'].lower(), '~')\n",
    "    ## remove reminder text (in parentheses)\n",
    "    text = re.sub(r'\\([^)]+\\)', '', text)\n",
    "    ## remove costs\n",
    "    text = re.sub(r'\\{[^}]+\\}', '', text)\n",
    "    ## genericize all p/t (de)buffs\n",
    "    text = re.sub(r'([+-])[\\dX*]/([+-])[\\dX*]', r'\\1X/\\2X', text)\n",
    "    ## genericize numbers\n",
    "    text = re.sub(r'\\d+', 'N', text)\n",
    "    ## split on punctuation and spaces\n",
    "    tokens = re.split(r'[\\s.,;:—()]+', text)\n",
    "    # use only unique tokens?\n",
    "    # tokens = set(tokens)\n",
    "    # stem tokens\n",
    "    tokens = (stemmer.stem(t) for t in tokens if t and t not in stopwords)\n",
    "\n",
    "    ## The following allows us to singularize certain terms.\n",
    "    ## For example, the word 'equip' is way over-represented on equipment\n",
    "    counter = collections.Counter(tokens)\n",
    "    if counter['equip']:\n",
    "        counter['equip'] = 1\n",
    "    \n",
    "    tokens = itertools.chain.from_iterable([token] * count for token, count in counter.items())\n",
    "        \n",
    "    return list(tokens)\n",
    "\n",
    "documents = [tokenize(c) for c in cards.values()]\n",
    "# next(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
      "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
     ]
    }
   ],
   "source": [
    "dictionary = corpora.Dictionary(documents)\n",
    "dictionary.save('dictionary.dict')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1917\n"
     ]
    }
   ],
   "source": [
    "print(len(dictionary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = [dictionary.doc2bow(doc) for doc in documents]\n",
    "corpora.MmCorpus.serialize('card_text_corpus.mm', corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model - LSI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = models.TfidfModel(corpus)\n",
    "corpus_tfidf = tfidf[corpus]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "lsi = models.LsiModel(corpus, id2word=dictionary, num_topics=100)\n",
    "corpus_lsi = lsi[corpus_tfidf]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.537*\"creatur\" + 0.415*\"~\" + 0.257*\"target\" + 0.253*\"card\" + 0.219*\"control\" + 0.179*\"turn\" + 0.170*\"battlefield\" + 0.132*\"end\" + 0.131*\"n\" + 0.131*\"+x/+x\"'),\n",
       " (1,\n",
       "  '-0.576*\"card\" + 0.506*\"creatur\" + -0.255*\"librari\" + -0.183*\"~\" + -0.177*\"put\" + -0.153*\"player\" + -0.143*\"hand\" + 0.140*\"turn\" + 0.139*\"enchant\" + -0.132*\"battlefield\"'),\n",
       " (2,\n",
       "  '-0.649*\"~\" + 0.411*\"card\" + 0.381*\"creatur\" + 0.185*\"librari\" + -0.178*\"counter\" + -0.158*\"enter\" + -0.142*\"damag\" + -0.133*\"battlefield\" + 0.131*\"enchant\" + -0.127*\"n\"'),\n",
       " (3,\n",
       "  '-0.407*\"target\" + -0.376*\"damag\" + -0.349*\"n\" + -0.307*\"player\" + -0.293*\"deal\" + 0.284*\"battlefield\" + 0.197*\"enter\" + 0.188*\"+x/+x\" + 0.171*\"~\" + 0.156*\"counter\"'),\n",
       " (4,\n",
       "  '-0.577*\"turn\" + -0.473*\"end\" + -0.282*\"get\" + 0.265*\"creatur\" + 0.222*\"enchant\" + -0.215*\"+x/+x\" + -0.161*\"gain\" + 0.160*\"damag\" + 0.153*\"deal\" + 0.131*\"n\"')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lsi.print_topics()[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Each player discards their hand, then draws cards equal to the greatest number of cards a player discarded this way.'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "card = cards['Windfall']\n",
    "card['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 1.026754946131473),\n",
       " (1, -1.8650618411934503),\n",
       " (2, 1.0913062891403233),\n",
       " (3, -0.6766340059751109),\n",
       " (4, -0.06561613951360196)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec_bow = dictionary.doc2bow(tokenize(card))\n",
    "vec_lsi = lsi[vec_bow]\n",
    "vec_lsi[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = similarities.MatrixSimilarity(corpus_lsi)\n",
    "index.save('all_cards_lsi.index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def similarity(card):\n",
    "    vec_bow = dictionary.doc2bow(tokenize(card))\n",
    "    vec_lsi = lsi[tfidf[vec_bow]]\n",
    "    scores = index[vec_lsi]\n",
    "    return sorted(enumerate(scores),\n",
    "            key=itemgetter(1), reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(19418, 1.0),\n",
       " (19306, 0.9999841),\n",
       " (8673, 0.94538784),\n",
       " (9486, 0.8555135),\n",
       " (9504, 0.8487427),\n",
       " (16923, 0.84758043),\n",
       " (10650, 0.82489854),\n",
       " (487, 0.8227881),\n",
       " (6156, 0.80599296),\n",
       " (13493, 0.8019552)]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity(card)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_similar_cards(card_name, N=10):\n",
    "    card = cards[card_name]\n",
    "    similarity_scores = similarity(card)\n",
    "    similar_cards = []\n",
    "    for name_idx, score in similarity_scores:\n",
    "        this_card_name = card_names[name_idx]\n",
    "        if this_card_name != card_name:\n",
    "            this_card = cards[this_card_name]\n",
    "            similar_cards.append((score,\n",
    "                    ' '.join([this_card['name'], this_card.get('manaCost', '')]),\n",
    "                    this_card['text'],\n",
    "                    ))\n",
    "        if len(similar_cards) >= N:\n",
    "            break\n",
    "    return similar_cards\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Query similar cards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.80681056, 'Inner Fire {3}{R}', 'Add {R} for each card in your hand.'),\n",
       " (0.769855,\n",
       "  'Serum Powder {3}',\n",
       "  '{T}: Add {C}.\\nAny time you could mulligan and Serum Powder is in your hand, you may exile all the cards from your hand, then draw that many cards. (You can do this in addition to taking mulligans.)'),\n",
       " (0.7496449,\n",
       "  'Simian Spirit Guide {2}{R}',\n",
       "  'Exile Simian Spirit Guide from your hand: Add {R}.'),\n",
       " (0.74501336,\n",
       "  'Reliquary Tower ',\n",
       "  'You have no maximum hand size.\\n{T}: Add {C}.'),\n",
       " (0.74501336,\n",
       "  'Thought Vessel {2}',\n",
       "  'You have no maximum hand size.\\n{T}: Add {C}.'),\n",
       " (0.7206175,\n",
       "  'Chrome Mox {0}',\n",
       "  \"Imprint — When Chrome Mox enters the battlefield, you may exile a nonartifact, nonland card from your hand.\\n{T}: Add one mana of any of the exiled card's colors.\"),\n",
       " (0.7045518,\n",
       "  'Scavenger Grounds ',\n",
       "  '{T}: Add {C}.\\n{2}, {T}, Sacrifice a Desert: Exile all cards from all graveyards.'),\n",
       " (0.70360094,\n",
       "  'Struggle for Sanity {2}{B}{B}',\n",
       "  'Target opponent reveals their hand. That player exiles a card from it, then you exile a card from it. Repeat this process until all cards in that hand have been exiled. That player returns the cards they exiled this way to their hand and puts the rest into their graveyard.'),\n",
       " (0.69269866,\n",
       "  'Bane Alley Broker {1}{U}{B}',\n",
       "  \"{T}: Draw a card, then exile a card from your hand face down.\\nYou may look at cards exiled with Bane Alley Broker.\\n{U}{B}, {T}: Return a card exiled with Bane Alley Broker to its owner's hand.\"),\n",
       " (0.6921438,\n",
       "  'Runic Repetition {2}{U}',\n",
       "  'Return target exiled card with flashback you own to your hand.')]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_similar_cards('Cadaverous Bloom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Living weapon (When this Equipment enters the battlefield, create a 0/0 black Germ creature token, then attach this to it.)\\nEquipped creature gets +X/+X, where X is the number of creature cards in all graveyards.\\nEquip {2}'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cards['Bonehoard']['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['live',\n",
       " 'weapon',\n",
       " 'equip',\n",
       " 'creatur',\n",
       " 'creatur',\n",
       " 'get',\n",
       " '+x/+x',\n",
       " 'x',\n",
       " 'number',\n",
       " 'card',\n",
       " 'graveyard']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenize(cards['Bonehoard'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model - LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "lda = models.LdaMulticore(corpus, id2word=dictionary, num_topics=100)\n",
    "corpus_lda = lda[corpus]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = similarities.MatrixSimilarity(corpus_lda)\n",
    "index.save('all_cards_lda.index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def similarity_lda(card):\n",
    "    vec_bow = dictionary.doc2bow(tokenize(card))\n",
    "    vec_lda = lda[vec_bow]\n",
    "    scores = index[vec_lda]\n",
    "    return sorted(enumerate(scores),\n",
    "            key=itemgetter(1), reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_similar_cards_lda(card_name, N=10):\n",
    "    card = cards[card_name]\n",
    "    similarity_scores = similarity_lda(card)\n",
    "    similar_cards = []\n",
    "    for name_idx, score in similarity_scores:\n",
    "        this_card_name = card_names[name_idx]\n",
    "        if this_card_name != card_name:\n",
    "            this_card = cards[this_card_name]\n",
    "            similar_cards.append((score, this_card['name'], this_card.get('text', '')))\n",
    "        if len(similar_cards) >= N:\n",
    "            break\n",
    "    return similar_cards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1.0,\n",
       "  '\"Rumors of My Death . . .\"',\n",
       "  '{3}{B}, Exile a permanent you control with a League of Dastardly Doom watermark: Return a permanent card with a League of Dastardly Doom watermark from your graveyard to the battlefield.'),\n",
       " (1.0,\n",
       "  'Airdrop Aeronauts',\n",
       "  'Flying\\nRevolt — When Airdrop Aeronauts enters the battlefield, if a permanent you controlled left the battlefield this turn, you gain 5 life.'),\n",
       " (1.0,\n",
       "  'Anafenza, Kin-Tree Spirit',\n",
       "  'Whenever another nontoken creature enters the battlefield under your control, bolster 1. (Choose a creature with the least toughness among creatures you control and put a +1/+1 counter on it.)'),\n",
       " (1.0,\n",
       "  'Arcbound Bruiser',\n",
       "  'Modular 3 (This enters the battlefield with three +1/+1 counters on it. When it dies, you may put its +1/+1 counters on target artifact creature.)'),\n",
       " (1.0,\n",
       "  'Arcbound Crusher',\n",
       "  'Trample\\nWhenever another artifact enters the battlefield, put a +1/+1 counter on Arcbound Crusher.\\nModular 1 (This enters the battlefield with a +1/+1 counter on it. When it dies, you may put its +1/+1 counters on target artifact creature.)'),\n",
       " (1.0,\n",
       "  'Arcbound Overseer',\n",
       "  'At the beginning of your upkeep, put a +1/+1 counter on each creature with modular you control.\\nModular 6 (This enters the battlefield with six +1/+1 counters on it. When it dies, you may put its +1/+1 counters on target artifact creature.)'),\n",
       " (1.0,\n",
       "  'Arcbound Ravager',\n",
       "  'Sacrifice an artifact: Put a +1/+1 counter on Arcbound Ravager.\\nModular 1 (This enters the battlefield with a +1/+1 counter on it. When it dies, you may put its +1/+1 counters on target artifact creature.)'),\n",
       " (1.0,\n",
       "  \"Ashnod's Transmogrant\",\n",
       "  \"{T}, Sacrifice Ashnod's Transmogrant: Put a +1/+1 counter on target nonartifact creature. That creature becomes an artifact in addition to its other types.\"),\n",
       " (1.0,\n",
       "  'Baron Von Count',\n",
       "  'Baron Von Count enters the battlefield with a doom counter on \"5.\"\\nWhenever you cast a spell with the indicated numeral in its mana cost, text box, power, or toughness, move the doom counter one numeral to the left.\\nWhen the doom counter moves from \"1,\" destroy target player and put that doom counter on \"5.\"'),\n",
       " (1.0,\n",
       "  'Bosh, Iron Golem',\n",
       "  \"Trample\\n{3}{R}, Sacrifice an artifact: Bosh, Iron Golem deals damage equal to the sacrificed artifact's converted mana cost to any target.\")]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_similar_cards_lda('Fatal Push')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['destroy',\n",
       " 'destroy',\n",
       " 'target',\n",
       " 'creatur',\n",
       " 'creatur',\n",
       " 'convert',\n",
       " 'convert',\n",
       " 'mana',\n",
       " 'mana',\n",
       " 'cost',\n",
       " 'cost',\n",
       " 'n',\n",
       " 'n',\n",
       " 'less',\n",
       " 'less',\n",
       " 'revolt',\n",
       " 'instead',\n",
       " 'perman',\n",
       " 'control',\n",
       " 'left',\n",
       " 'battlefield',\n",
       " 'turn']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenize(cards['Fatal Push'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colorIdentity': ['B'],\n",
       " 'colors': ['B'],\n",
       " 'convertedManaCost': 1.0,\n",
       " 'foreignData': [],\n",
       " 'layout': 'normal',\n",
       " 'legalities': {'commander': 'Legal',\n",
       "  'duel': 'Legal',\n",
       "  'frontier': 'Legal',\n",
       "  'legacy': 'Legal',\n",
       "  'modern': 'Legal',\n",
       "  'vintage': 'Legal'},\n",
       " 'manaCost': '{B}',\n",
       " 'name': 'Fatal Push',\n",
       " 'printings': ['AER', 'F17', 'PRM'],\n",
       " 'rulings': [{'date': '2017-02-09',\n",
       "   'text': 'Fatal Push can target any creature, even one with converted mana cost 5 or greater. The creature’s converted mana cost is checked only as Fatal Push resolves.'},\n",
       "  {'date': '2017-02-09',\n",
       "   'text': 'If the mana cost of a creature on the battlefield includes {X}, X is considered to be 0.'},\n",
       "  {'date': '2017-02-09',\n",
       "   'text': 'Revolt abilities check only whether a permanent you controlled left the battlefield this turn or not. They don’t apply multiple times if more than one permanent you controlled left the battlefield. They don’t check whether the permanent that left the battlefield is still in the zone it moved to.'},\n",
       "  {'date': '2017-02-09',\n",
       "   'text': 'Revolt abilities don’t care why the permanent left the battlefield, who caused it to move, or where it moved to. They’re equally satisfied by an artifact you sacrificed to pay a cost, a creature you controlled that was destroyed by Murder, or an enchantment you returned to your hand with Leave in the Dust.'},\n",
       "  {'date': '2017-02-09',\n",
       "   'text': 'Tokens that leave the battlefield will satisfy a revolt ability.'},\n",
       "  {'date': '2017-02-09',\n",
       "   'text': 'Energy counters aren’t permanents. Paying {E} won’t satisfy a revolt ability.'}],\n",
       " 'scryfallOracleId': '16437a83-be52-44cd-a768-a767c9347eb2',\n",
       " 'subtypes': [],\n",
       " 'supertypes': [],\n",
       " 'text': 'Destroy target creature if it has converted mana cost 2 or less.\\nRevolt — Destroy that creature if it has converted mana cost 4 or less instead if a permanent you controlled left the battlefield this turn.',\n",
       " 'type': 'Instant',\n",
       " 'types': ['Instant'],\n",
       " 'uuid': 'c3d2d3d2-4c1e-5095-a978-4053589a9c42'}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cards['Fatal Push']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model - HDP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hdp = models.HdpModel(corpus, id2word=dictionary)\n",
    "corpus_hdp = hdp[corpus]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = similarities.MatrixSimilarity(corpus_hdp)\n",
    "index.save('all_cards_hdp.index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def similarity_hdp(card):\n",
    "    vec_bow = dictionary.doc2bow(tokenize(card))\n",
    "    vec_hdp = hdp[vec_bow]\n",
    "    scores = index[vec_hdp]\n",
    "    return sorted(enumerate(scores),\n",
    "            key=itemgetter(1), reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_similar_cards_hdp(card_name, N=10):\n",
    "    card = cards[card_name]\n",
    "    similarity_scores = similarity_hdp(card)\n",
    "    similar_cards = []\n",
    "    for name_idx, score in similarity_scores:\n",
    "        this_card_name = card_names[name_idx]\n",
    "        if this_card_name != card_name:\n",
    "            this_card = cards[this_card_name]\n",
    "            similar_cards.append((score, this_card['name'], this_card.get('text', '')))\n",
    "        if len(similar_cards) >= N:\n",
    "            break\n",
    "    return similar_cards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_similar_cards_hdp('Windfall')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdp.print_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}