Dataset Recognition (#79)

* added directory structure to docs * split parsers into separate files, added validation * improved import guard * dataset recognition * updated notebook example * improved coco parser * cleaned parsers * remove prints * added save dir option * changed single file download * parser test cases * docs * removed dataset deleting * requirements * added logging to the end * Update create_ml_parser.py * added gcloud authentication * pre-commit formatting * [Automated] Updated coverage badge * updated for latest LuxonisFileSystem * changed test parsers save path * [Automated] Updated coverage badge --------- Co-authored-by: GitHub Actions <[email protected]>
luxonis · Mar 2, 2024 · eb5568e · eb5568e
1 parent a5fb23d
commit eb5568e
Show file tree

Hide file tree

Showing 18 changed files with 1,760 additions and 1,155 deletions.
diff --git a/examples/Data_Parser_Example.ipynb b/examples/Data_Parser_Example.ipynb
@@ -17,29 +17,15 @@
    "source": [
     "import os\n",
     "import zipfile\n",
+    "from pathlib import Path\n",
     "\n",
     "import cv2\n",
     "import gdown\n",
     "import matplotlib.pyplot as plt\n",
     "import numpy as np\n",
     "\n",
-    "from luxonis_ml.data import LuxonisDataset, LuxonisLoader, LuxonisParser\n",
-    "from luxonis_ml.enums import DatasetType, LabelType"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e5a3a45c-7152-41a8-9ebf-db54cb84edcc",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Delete dataset if exists\n",
-    "\n",
-    "dataset_name = \"coco_test\"\n",
-    "if LuxonisDataset.exists(dataset_name):\n",
-    "    dataset = LuxonisDataset(dataset_name)\n",
-    "    dataset.delete_dataset()"
+    "from luxonis_ml.data import LuxonisLoader, LuxonisParser\n",
+    "from luxonis_ml.enums import LabelType"
    ]
   },
   {
@@ -61,20 +47,20 @@
    "source": [
     "url = \"https://drive.google.com/uc?id=1XlvFK7aRmt8op6-hHkWVKIJQeDtOwoRT\"\n",
     "output_zip = \"../data/COCO_people_subset.zip\"\n",
-    "output_folder = \"../data/\"\n",
+    "dataset_dir = \"../data/coco_test\"\n",
     "\n",
-    "if not os.path.exists(output_folder):\n",
-    "    os.mkdir(output_folder)\n",
+    "if not os.path.exists(dataset_dir):\n",
+    "    Path(dataset_dir).mkdir(parents=True)\n",
     "# Check if the data already exists\n",
     "if not os.path.exists(output_zip) and not os.path.exists(\n",
-    "    os.path.join(output_folder, \"COCO_people_subset\")\n",
+    "    os.path.join(dataset_dir, \"COCO_people_subset\")\n",
     "):\n",
     "    # Download the file\n",
     "    gdown.download(url, output_zip, quiet=False)\n",
     "\n",
     "    # Unzip the file\n",
     "    with zipfile.ZipFile(output_zip, \"r\") as zip_ref:\n",
-    "        zip_ref.extractall(output_folder)\n",
+    "        zip_ref.extractall(dataset_dir)\n",
     "else:\n",
     "    print(\"Data already exists. Exiting.\")"
    ]
@@ -102,13 +88,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "parser = LuxonisParser(dataset_name=dataset_name)\n",
-    "dataset = parser.parse_raw_dir(\n",
-    "    DatasetType.COCO,\n",
-    "    random_split=True,\n",
-    "    image_dir=os.path.join(output_folder, \"person_val2017_subset\"),\n",
-    "    annotation_path=os.path.join(output_folder, \"person_keypoints_val2017.json\"),\n",
-    ")"
+    "dataset_name = \"coco_test\"\n",
+    "parser = LuxonisParser(dataset_dir, dataset_name=dataset_name, delete_existing=True)\n",
+    "dataset = parser.parse(random_split=True)"
    ]
   },
   {
@@ -174,7 +156,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.8.18"
   }
  },
  "nbformat": 4,