Fixed data loading pipeline and projection dimension bug

Author: kharshith-k

examples/vision/ipynb/object_detection_using_vision_transformer.ipynb

@@ -2,6 +2,7 @@
  "cells": [
   {
    "cell_type": "markdown",
+   "id": "39db6022",
    "metadata": {
     "colab_type": "text"
    },
@@ -16,6 +17,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "4baf89fb",
    "metadata": {
     "colab_type": "text"
    },
@@ -36,6 +38,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "021c1f47",
    "metadata": {
     "colab_type": "text"
    },
@@ -46,6 +49,7 @@
   {
    "cell_type": "code",
    "execution_count": 0,
+   "id": "1a686205",
    "metadata": {
     "colab_type": "code"
    },
@@ -70,6 +74,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "4ded145b",
    "metadata": {
     "colab_type": "text"
    },
@@ -82,6 +87,7 @@
   {
    "cell_type": "code",
    "execution_count": 0,
+   "id": "37944a37",
    "metadata": {
     "colab_type": "code"
    },
@@ -102,10 +108,10 @@
     "\n",
     "# Extracting tar files found inside main zip file\n",
     "shutil.unpack_archive(\n",
-    "    os.path.join(download_base_dir, \"caltech-101\", \"101_ObjectCategories.tar.gz\"), \".\"\n",
+    "    os.path.join(download_base_dir, \"caltech_101_zipped\", \"caltech-101\", \"101_ObjectCategories.tar.gz\"), \".\"\n",
     ")\n",
     "shutil.unpack_archive(\n",
-    "    os.path.join(download_base_dir, \"caltech-101\", \"Annotations.tar\"), \".\"\n",
+    "    os.path.join(download_base_dir, \"caltech_101_zipped\", \"caltech-101\", \"Annotations.tar\"), \".\"\n",
     ")\n",
     "\n",
     "# list of paths to images and annotations\n",
@@ -165,6 +171,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "60309cdb",
    "metadata": {
     "colab_type": "text"
    },
@@ -179,6 +186,7 @@
   {
    "cell_type": "code",
    "execution_count": 0,
+   "id": "5f584e87",
    "metadata": {
     "colab_type": "code"
    },
@@ -193,6 +201,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "e8b8d2e1",
    "metadata": {
     "colab_type": "text"
    },
@@ -203,6 +212,7 @@
   {
    "cell_type": "code",
    "execution_count": 0,
+   "id": "387f4dfe",
    "metadata": {
     "colab_type": "code"
    },
@@ -240,6 +250,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "e6b5b2c0",
    "metadata": {
     "colab_type": "text"
    },
@@ -250,6 +261,7 @@
   {
    "cell_type": "code",
    "execution_count": 0,
+   "id": "ef90f17d",
    "metadata": {
     "colab_type": "code"
    },
@@ -278,6 +290,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "bc775a39",
    "metadata": {
     "colab_type": "text"
    },
@@ -292,6 +305,7 @@
   {
    "cell_type": "code",
    "execution_count": 0,
+   "id": "096d74d8",
    "metadata": {
     "colab_type": "code"
    },
@@ -334,6 +348,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "54c4ba4e",
    "metadata": {
     "colab_type": "text"
    },
@@ -352,6 +367,7 @@
   {
    "cell_type": "code",
    "execution_count": 0,
+   "id": "a61642a3",
    "metadata": {
     "colab_type": "code"
    },
@@ -366,12 +382,13 @@
     "    transformer_units,\n",
     "    transformer_layers,\n",
     "    mlp_head_units,\n",
+    "    hidden_dim,\n",
     "):\n",
     "    inputs = keras.Input(shape=input_shape)\n",
     "    # Create patches\n",
     "    patches = Patches(patch_size)(inputs)\n",
     "    # Encode patches\n",
-    "    encoded_patches = PatchEncoder(num_patches, projection_dim)(patches)\n",
+    "    encoded_patches = PatchEncoder(num_patches, hidden_dim)(patches)\n",
     "\n",
     "    # Create multiple layers of the Transformer block.\n",
     "    for _ in range(transformer_layers):\n",
@@ -407,6 +424,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "43dcd366",
    "metadata": {
     "colab_type": "text"
    },
@@ -417,6 +435,7 @@
   {
    "cell_type": "code",
    "execution_count": 0,
+   "id": "3751e9ba",
    "metadata": {
     "colab_type": "code"
    },
@@ -461,10 +480,11 @@
     "num_patches = (image_size // patch_size) ** 2\n",
     "projection_dim = 64\n",
     "num_heads = 4\n",
+    "hidden_dim = projection_dim * num_heads\n",
     "# Size of the transformer layers\n",
     "transformer_units = [\n",
-    "    projection_dim * 2,\n",
-    "    projection_dim,\n",
+    "    hidden_dim * 2,\n",
+    "    hidden_dim,\n",
     "]\n",
     "transformer_layers = 4\n",
     "mlp_head_units = [2048, 1024, 512, 64, 32]  # Size of the dense layers\n",
@@ -482,6 +502,7 @@
     "    transformer_units,\n",
     "    transformer_layers,\n",
     "    mlp_head_units,\n",
+    "    hidden_dim\n",
     ")\n",
     "\n",
     "# Train model\n",
@@ -506,6 +527,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "3bde0c6b",
    "metadata": {
     "colab_type": "text"
    },
@@ -516,6 +538,7 @@
   {
    "cell_type": "code",
    "execution_count": 0,
+   "id": "dd6fcf2a",
    "metadata": {
     "colab_type": "code"
    },
@@ -643,6 +666,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "72ff599a",
    "metadata": {
     "colab_type": "text"
    },
@@ -655,6 +679,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "2d60f09c",
    "metadata": {},
    "source": [
     "## Relevant Chapters from Deep Learning with Python\n",

examples/vision/md/object_detection_using_vision_transformer.md

@@ -68,10 +68,10 @@ download_base_dir = os.path.dirname(path_to_downloaded_file)
 
 # Extracting tar files found inside main zip file
 shutil.unpack_archive(
-    os.path.join(download_base_dir, "caltech-101", "101_ObjectCategories.tar.gz"), "."
+    os.path.join(download_base_dir, "caltech_101_zipped", "caltech-101", "101_ObjectCategories.tar.gz"), "."
 )
 shutil.unpack_archive(
-    os.path.join(download_base_dir, "caltech-101", "Annotations.tar"), "."
+    os.path.join(download_base_dir, "caltech_101_zipped", "caltech-101", "Annotations.tar"), "."
 )
 
 # list of paths to images and annotations
@@ -299,12 +299,13 @@ def create_vit_object_detector(
     transformer_units,
     transformer_layers,
     mlp_head_units,
+    hidden_dim,
 ):
     inputs = keras.Input(shape=input_shape)
     # Create patches
     patches = Patches(patch_size)(inputs)
     # Encode patches
-    encoded_patches = PatchEncoder(num_patches, projection_dim)(patches)
+    encoded_patches = PatchEncoder(num_patches, hidden_dim)(patches)
 
     # Create multiple layers of the Transformer block.
     for _ in range(transformer_layers):
@@ -384,10 +385,11 @@ num_epochs = 100
 num_patches = (image_size // patch_size) ** 2
 projection_dim = 64
 num_heads = 4
+hidden_dim = projection_dim * num_heads
 # Size of the transformer layers
 transformer_units = [
-    projection_dim * 2,
-    projection_dim,
+    hidden_dim * 2,
+    hidden_dim,
 ]
 transformer_layers = 4
 mlp_head_units = [2048, 1024, 512, 64, 32]  # Size of the dense layers
@@ -405,6 +407,7 @@ vit_object_detector = create_vit_object_detector(
     transformer_units,
     transformer_layers,
     mlp_head_units,
+    hidden_dim,
 )
 
 # Train model

examples/vision/object_detection_using_vision_transformer.py

@@ -63,10 +63,19 @@
 
 # Extracting tar files found inside main zip file
 shutil.unpack_archive(
-    os.path.join(download_base_dir, "caltech-101", "101_ObjectCategories.tar.gz"), "."
+    os.path.join(
+        download_base_dir,
+        "caltech_101_zipped",
+        "caltech-101",
+        "101_ObjectCategories.tar.gz",
+    ),
+    ".",
 )
 shutil.unpack_archive(
-    os.path.join(download_base_dir, "caltech-101", "Annotations.tar"), "."
+    os.path.join(
+        download_base_dir, "caltech_101_zipped", "caltech-101", "Annotations.tar"
+    ),
+    ".",
 )
 
 # list of paths to images and annotations
@@ -264,12 +273,13 @@ def create_vit_object_detector(
     transformer_units,
     transformer_layers,
     mlp_head_units,
+    hidden_dim,
 ):
     inputs = keras.Input(shape=input_shape)
     # Create patches
     patches = Patches(patch_size)(inputs)
     # Encode patches
-    encoded_patches = PatchEncoder(num_patches, projection_dim)(patches)
+    encoded_patches = PatchEncoder(num_patches, hidden_dim)(patches)
 
     # Create multiple layers of the Transformer block.
     for _ in range(transformer_layers):
@@ -347,10 +357,11 @@ def run_experiment(model, learning_rate, weight_decay, batch_size, num_epochs):
 num_patches = (image_size // patch_size) ** 2
 projection_dim = 64
 num_heads = 4
+hidden_dim = projection_dim * num_heads
 # Size of the transformer layers
 transformer_units = [
-    projection_dim * 2,
-    projection_dim,
+    hidden_dim * 2,
+    hidden_dim,
 ]
 transformer_layers = 4
 mlp_head_units = [2048, 1024, 512, 64, 32]  # Size of the dense layers
@@ -368,6 +379,7 @@ def run_experiment(model, learning_rate, weight_decay, batch_size, num_epochs):
     transformer_units,
     transformer_layers,
     mlp_head_units,
+    hidden_dim,
 )
 
 # Train model
@@ -389,7 +401,6 @@ def plot_history(item):
 
 plot_history("loss")
 
-
 """
 ## Evaluate the model
 """