data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import csv\n",
    "import glob\n",
    "\n",
    "## Download data from here: https://www.kaggle.com/c/dog-breed-identification\n",
    "\n",
    "BASE_PATH = \"/Users/wctjerry/Documents/dataset/dog-breed-identification\"\n",
    "train_path = \"train\"\n",
    "map_csv_path = \"labels.csv\"\n",
    "\n",
    "## Collect all files' path for images in train folder\n",
    "train_all_files_path = glob.glob(os.path.join(BASE_PATH, train_path + \"/*.jpg\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Create id - breed pairs as well as breed - ids pairs \n",
    "## The labels.csv file contains two columns: id and corresponding breed\n",
    "\n",
    "id_breed_dict = {}\n",
    "\n",
    "reader = csv.DictReader(os.path.join(BASE_PATH, map_csv_path))\n",
    "with open(os.path.join(BASE_PATH, map_csv_path)) as f:\n",
    "    reader = csv.DictReader(f)\n",
    "    for r in reader:\n",
    "        id_breed_dict[r['id']] = r['breed']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def move_file(file_path):\n",
    "    ## Move the image to corresponding breed folder. Create the breed folder if not exists\n",
    "    ##\n",
    "    ## INPUT: A single image file's path. The file name is its id. \n",
    "    ## OUTPUT: None\n",
    "    \n",
    "    image_id = os.path.basename(file_path).split('.')[0]\n",
    "    breed = id_breed_dict[image_id]\n",
    "    target_path = os.path.join(os.path.dirname(file_path), breed)\n",
    "    \n",
    "    if not os.path.exists(target_path) :\n",
    "        os.mkdir(target_path)\n",
    "        \n",
    "    os.rename(str(file_path), os.path.join(target_path, os.path.basename(file_path)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Iterate to move images so as to meet the Input structure requirement of Create ML\n",
    "##\n",
    "## Final structure is like:\n",
    "##   affenpinscher\n",
    "##   -- 00ca18751837cd6a22813f8e221f7819.jpg\n",
    "##   -- 1a012faf98b32039adc17ee708bcb360.jpg\n",
    "##   -- 1aa98488ce65f9b57b468ccde4f1cad8.jpg\n",
    "##   -- ...\n",
    "##   afghan_hound\n",
    "##   -- 0a4f1e17d720cdff35814651402b7cf4.jpg\n",
    "##   -- 0d5a88f0ab2db8d34b533c69768135e8.jpg\n",
    "##   -- ...\n",
    "##   ...\n",
    "\n",
    "for file_path in train_all_files_path:\n",
    "    move_file(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Extract 5% images from each breed folder and move to test path\n",
    "## Although there is test data in the original dataset, there is no labels there.\n",
    "## So create this 5% test data for testing use and AI VS. Human use.\n",
    "\n",
    "import random\n",
    "\n",
    "validation = \"test_generated\"\n",
    "validation_path = os.path.join(BASE_PATH, validation)\n",
    "breeds = set(id_breed_dict.values())\n",
    "id_breed_validation_dict = {}\n",
    "\n",
    "if not os.path.exists(validation_path) :\n",
    "        os.mkdir(validation_path)\n",
    "\n",
    "for breed in breeds:\n",
    "    all_file_path_by_breed = glob.glob(os.path.join(BASE_PATH, train_path, breed + \"/*.jpg\"))\n",
    "    validation_breed = random.sample(all_file_path_by_breed, int(len(all_file_path_by_breed) * 0.05))\n",
    "\n",
    "    if not os.path.exists(os.path.join(validation_path, breed)) :\n",
    "        os.mkdir(os.path.join(validation_path, breed))\n",
    "    \n",
    "    for v in validation_breed:\n",
    "        i = os.path.basename(v).split('.')[0]\n",
    "        id_breed_validation_dict[i] = id_breed_dict[i]\n",
    "        os.rename(str(v), os.path.join(BASE_PATH, validation, breed, os.path.basename(v)))\n",
    "        \n",
    "\n",
    "f = open(os.path.join(validation_path, \"test_generated_dict.txt\"), \"w\")\n",
    "f.write(str(id_breed_validation_dict))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}