FINISHED 07a Exquisite Corpse - Preparing the Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "pandas.set_option('display.max_colwidth', 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrubLine(line):\n",
    "    newline = line.replace('Mrs.', ' ')\n",
    "    newline = newline.replace('Mr.', ' ')\n",
    "    newline = newline.replace('Ms.', ' ')\n",
    "    newline = newline.replace(' m.', ' ')\n",
    "    newline = newline.replace('R.', ' ')\n",
    "    newline = newline.replace('W.', ' ')\n",
    "    newline = newline.replace('\"', '')\n",
    "    newline = newline.replace('“', '')\n",
    "    newline = newline.replace('”', '')\n",
    "    return newline\n",
    "\n",
    "\n",
    "with open(\"data/raw frankenstein.txt\", \"rt\") as fin:\n",
    "    with open(\"data/frankenstein.txt\", \"wt\") as fout:\n",
    "        for line in fin:\n",
    "            newline = scrubLine(line)\n",
    "            fout.write(newline)\n",
    "\n",
    "with open(\"data/raw pride.txt\", \"rt\") as fin:\n",
    "    with open(\"data/pride.txt\", \"wt\") as fout:\n",
    "        for line in fin:\n",
    "            newline = scrubLine(line)\n",
    "            fout.write(newline)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total frankenstein sentences:  2000\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "with open('data/frankenstein.txt') as f:\n",
    "    text = f.read()    \n",
    "    sentences = re.split(r' *[\\.\\!\\?][\\'\"\\)\\]]* *', text)[0:2000]\n",
    "    \n",
    "    for i in range(len(sentences)):\n",
    "        sentences[i] = sentences[i].replace(\"\\n\", \" \") + \".\"\n",
    "        \n",
    "print(\"total frankenstein sentences: \", len(sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total Pride & Prejudice sentences:  2000\n"
     ]
    }
   ],
   "source": [
    "with open('data/pride.txt') as f:\n",
    "    text = f.read()    \n",
    "    pride_sentences = re.split(r' *[\\.\\!][\\'\"\\)\\]]* *', text)[0:2000]\n",
    "    \n",
    "    for i in range(len(pride_sentences)):\n",
    "        pride_sentences[i] = pride_sentences[i].replace(\"\\n\", \" \") + \".\"\n",
    "        \n",
    "print(\"total Pride & Prejudice sentences: \", len(pride_sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2000 usable sentences in Pride & Prejudice\n",
      "We have collected a total of 4000 questions for our Exquisite Corpse\n"
     ]
    }
   ],
   "source": [
    "previous_count = len(sentences)\n",
    "\n",
    "for line in pride_sentences:\n",
    "        sentences.append(line)\n",
    "        \n",
    "print(\"Found {} usable sentences in Pride & Prejudice\".format(len(sentences) - previous_count))\n",
    "print(\"We have collected a total of {} questions for our Exquisite Corpse\".format(len(sentences)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Frankenstein, or the Modern Prometheus  Letter 1  St.', 'Petersburgh, Dec.', '11th, 17--  TO   Saville, England  You will rejoice to hear that no disaster has accompanied the commencement of an enterprise which you have regarded with such evil forebodings.']\n",
      "['But still he would be her husband.', '  Felix had accidentally been present at the trial; his horror and indignation were uncontrollable when he heard the decision of the court.', 'If she is condemned, I never shall know joy more.']\n"
     ]
    }
   ],
   "source": [
    "# LET'S SHUFFLE UP THE SENTENCES \n",
    "\n",
    "from random import shuffle\n",
    "\n",
    "print(sentences[0:3])\n",
    "shuffle(sentences)\n",
    "print(sentences[0:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LET'S SAVE OUR RAW QUESTIONS AS TXT\n",
    "\n",
    "with open(\"data/sentences.txt\", \"wt\") as fout:\n",
    "    for i in range(len(sentences)):\n",
    "        fout.write(sentences[i])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LET'S SAVE OUR RAW QUESTIONS IN THE MORE USABLE 'CSV' FORMAT\n",
    "import csv\n",
    "import itertools\n",
    "\n",
    "grouped = zip(*[sentences])\n",
    "\n",
    "with open('data/sentences.csv', 'w') as out_file:\n",
    "    writer = csv.writer(out_file)\n",
    "    writer.writerow(['line'])\n",
    "    writer.writerows(grouped)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>line</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>But still he would be her husband.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Felix had accidentally been present at the trial; his horror and indignation were uncontrollable when he heard the decision of the court.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>If she is condemned, I never shall know joy more.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I have lived in the same house with her, at one time for five and at another for nearly two years.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I did not pretend to enter into the merits of the case, yet I inclined towards the opinions of the hero, whose extinction I wept, without precisely understanding it.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>But in giving an account of the progress of my intellect, I must not omit a circumstance which occurred in the beginning of the month of August of the same year.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>I am about to proceed on a long and difficult voyage, the emergencies of which will demand all my fortitude:  I am required not only to raise the spirits of others, but sometimes to sustain my own, when theirs are failing.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>In my education my father had taken the greatest precautions that my mind should be impressed with no supernatural horrors.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>But as it is, you must not let your fancy run away with you.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Bennet deigned not to make any reply, but, unable to contain herself, began scolding one of her daughters.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                             line\n",
       "0                                                                                                                                                                                              But still he would be her husband.\n",
       "1                                                                                       Felix had accidentally been present at the trial; his horror and indignation were uncontrollable when he heard the decision of the court.\n",
       "2                                                                                                                                                                               If she is condemned, I never shall know joy more.\n",
       "3                                                                                                                              I have lived in the same house with her, at one time for five and at another for nearly two years.\n",
       "4                                                           I did not pretend to enter into the merits of the case, yet I inclined towards the opinions of the hero, whose extinction I wept, without precisely understanding it.\n",
       "5                                                               But in giving an account of the progress of my intellect, I must not omit a circumstance which occurred in the beginning of the month of August of the same year.\n",
       "6  I am about to proceed on a long and difficult voyage, the emergencies of which will demand all my fortitude:  I am required not only to raise the spirits of others, but sometimes to sustain my own, when theirs are failing.\n",
       "7                                                                                                     In my education my father had taken the greatest precautions that my mind should be impressed with no supernatural horrors.\n",
       "8                                                                                                                                                                    But as it is, you must not let your fancy run away with you.\n",
       "9                                                                                                                      Bennet deigned not to make any reply, but, unable to contain herself, began scolding one of her daughters."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LET'S READ THE CSV FILE INTO PANDAS AND SEE THE FIRST 10 ENTRIES...\n",
    "lines = pandas.read_csv('data/sentences.csv', encoding='utf-8')\n",
    "lines[0:10]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}