{ "cells": [ { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "z-7UORN7Ymoo" }, "source": [ "Mount the drive to the instance." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "9X2hja-OpXM9" }, "outputs": [], "source": [ "from google.colab import drive\n", "drive.mount('/content/gdrive')" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "fjU2n68zYxU3" }, "source": [ "Import various modules.\n", "\n", "I used Tnesorflow and its Keras library as the backend. The Keras module is for preprocessing images. " ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": {}, "colab_type": "code", "id": "QIR81L-ArDn3" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[31mERROR: Could not find a version that satisfies the requirement tensorflow-gpu==2.0.0-alpha0\u001b[0m\r\n", "\u001b[31mERROR: No matching distribution found for tensorflow-gpu==2.0.0-alpha0\u001b[0m\r\n" ] }, { "ename": "ModuleNotFoundError", "evalue": "No module named 'joblib'", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_selection\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mto_categorical\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mModelCheckpoint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mReduceLROnPlateau\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mEarlyStopping\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/Documents/projects/TensorBird/tb-venv/lib/python3.8/site-packages/scikit_learn-0.24.1-py3.8-linux-x86_64.egg/sklearn/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0m_distributor_init\u001b[0m \u001b[0;31m# noqa: F401\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0m__check_build\u001b[0m \u001b[0;31m# noqa: F401\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 82\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mbase\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mclone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 83\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_show_versions\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mshow_versions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/Documents/projects/TensorBird/tb-venv/lib/python3.8/site-packages/scikit_learn-0.24.1-py3.8-linux-x86_64.egg/sklearn/base.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0m__version__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_config\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mget_config\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0m_IS_32BIT\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 18\u001b[0m from .utils._tags import (\n\u001b[1;32m 19\u001b[0m \u001b[0m_DEFAULT_TAGS\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/Documents/projects/TensorBird/tb-venv/lib/python3.8/site-packages/scikit_learn-0.24.1-py3.8-linux-x86_64.egg/sklearn/utils/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mmurmurhash\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmurmurhash3_32\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mclass_weight\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcompute_class_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompute_sample_weight\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 24\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0m_joblib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDataConversionWarning\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/Documents/projects/TensorBird/tb-venv/lib/python3.8/site-packages/scikit_learn-0.24.1-py3.8-linux-x86_64.egg/sklearn/utils/class_weight.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mvalidation\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0m_deprecate_positional_args\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/Documents/projects/TensorBird/tb-venv/lib/python3.8/site-packages/scikit_learn-0.24.1-py3.8-linux-x86_64.egg/sklearn/utils/validation.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;31m# mypy error: Module 'numpy.core.numeric' has no attribute 'ComplexWarning'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumeric\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mComplexWarning\u001b[0m \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mjoblib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mcontextlib\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msuppress\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'joblib'" ] } ], "source": [ "!pip install -q tensorflow-gpu==2.0.0-alpha0\n", "import cv2\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import matplotlib.patches as patches\n", "from __future__ import absolute_import, division, print_function\n", "import os\n", "import tensorflow as tf\n", "from sklearn.model_selection import train_test_split\n", "from keras.utils import to_categorical\n", "from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\n", "from keras.preprocessing import image\n", "keras = tf.keras\n", "datapath = '/content/gdrive/My Drive/Bird_ID_project/nabirds'" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "GVPN6iW0IvZ5" }, "source": [ "Reading in the pandas dataframe. It actually doesn't provide much beside the unique species class names." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "K99e_KEnkN4p" }, "outputs": [], "source": [ "train_frame = pd.read_csv(datapath+'/train.csv')\n", "val_frame = pd.read_csv(datapath+'/val.csv')\n", "test_frame = pd.read_csv(datapath+'/test.csv')\n", "# train_frame\n", "Bird_list=train_frame.class_name_sp.unique()\n", "Bird_id=train_frame.class_id_sp.unique()\n", "# print(Bird_list, Bird_id)\n", "Birds = dict(zip(Bird_id, Bird_list))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "R9pJGyS8b83K" }, "source": [ "Unzip the tar.gz files **to the Google Colab instance**. This drastically increased the training speed." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "cYGuyyUINUQ3" }, "outputs": [], "source": [ "!tar -xzf \"gdrive/My Drive/Bird_ID_project/nabirds/data.tar.gz\"\n", "!tar -xzf \"gdrive/My Drive/Bird_ID_project/nabirds/Darren_data.tar.gz\"" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "GWP0UQZdBDk1" }, "outputs": [], "source": [ "!mv /content/Darren_test/0289 /content/Darren_test/0867\n", "!mv /content/Darren_test/0095 /content/Darren_test/0553\n", "!rm -rf /content/Darren_test/NIL" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "0vRSm4DVI9Jd" }, "source": [ "Using Keras image generator to reduce RAM footprint (instead of using a large numpy array)." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "BxCuJxxxl2ge" }, "outputs": [], "source": [ "train_datagen = image.ImageDataGenerator(\n", " rescale=1./255,\n", "# width_shift_range=0.1,\n", "# height_shift_range=0.1,\n", "# zoom_range=0.1,\n", " fill_mode='constant',\n", " horizontal_flip=True,\n", " dtype=np.float32)\n", "\n", "val_datagen = image.ImageDataGenerator(\n", " rescale=1./255, \n", " dtype=np.float32)\n", "\n", "test_datagen = image.ImageDataGenerator(\n", " rescale=1./255,\n", " dtype=np.float32)\n", "\n", "train_generator = train_datagen.flow_from_directory(\n", " directory='/content/data/train/',\n", " #directory=datapath + '/data/train/',\n", " #classes=list(Bird_list),\n", " target_size=(224, 224),\n", " batch_size=32,\n", " class_mode='categorical')\n", "\n", "validation_generator = val_datagen.flow_from_directory(\n", " #directory=datapath + '/data/val/',\n", " directory='/content/data/val/',\n", " #classes=list(Bird_list),\n", " target_size=(224, 224),\n", " batch_size=32,\n", " class_mode='categorical')\n", "\n", "test_generator = test_datagen.flow_from_directory(\n", " #directory=datapath + '/data/test/',\n", " directory='/content/data/test/',\n", " shuffle=False,\n", " #classes=list(Bird_list),\n", " target_size=(224, 224),\n", " batch_size=32,\n", " class_mode='categorical')\n" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "HlsRpRXDMicF" }, "source": [ "Research has shown that starting from scratch will not work. So we use MobileNet V2 as our feature extractor. This is a relatively small network, which will help in the future when we want to run near real time inferences. We also add two layers: One global average and one prediction. " ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "ytj6wsAdM7Un" }, "outputs": [], "source": [ "IMG_SHAPE = (224,224,3)\n", "base_model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE,\n", " include_top=False, \n", " weights='imagenet')\n", "\n", "base_model.trainable = False\n", "\n", "global_average_layer = tf.keras.layers.GlobalAveragePooling2D()\n", "prediction_layer = keras.layers.Dense(404,activation='softmax')\n", "base_learning_rate = 0.0001\n", "model = tf.keras.Sequential([\n", " base_model,\n", " global_average_layer,\n", " prediction_layer\n", "])\n", "model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=base_learning_rate), \n", " loss='categorical_crossentropy', \n", " metrics=['accuracy'])\n", "model.summary()" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "bGi69hxZRSvy" }, "outputs": [], "source": [ "def fit_model(model, batch_size=32, epochs=10): \n", " history = model.fit_generator(\n", " generator=train_generator,\n", " steps_per_epoch=(len(train_frame) // batch_size),\n", " epochs=epochs,\n", " validation_data=validation_generator,\n", " callbacks=None\n", " )\n", " score = model.evaluate_generator(train_generator, verbose=1)\n", " probs = model.predict_generator(test_generator, verbose=1)\n", " return model, score, probs, history\n" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "dt80nu7nQlzm" }, "outputs": [], "source": [ "fit_model, score, probs, history = fit_model(model, batch_size=32, epochs=10)" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "YdfI5Fzw6CRx" }, "outputs": [], "source": [ "def top3(probs, GT):\n", " t3 = np.argsort(probs)[-3:]\n", " #print(t3)\n", " if GT in t3:\n", " return 1\n", " else:\n", " return 0\n", " \n", "def top5(probs, GT):\n", " t5 = np.argsort(probs)[-5:]\n", " if GT in t5:\n", " return 1\n", " else:\n", " return 0\n", " \n", "def top3_idx(probs):\n", " return np.flip(np.argsort(probs)[-3:],0), np.flip(probs[np.argsort(probs)[-3:]],0)\n", " #print(t3)\n", "\n", "def top5_idx(probs):\n", " return np.flip(np.argsort(probs)[-5:])\n", " #print(t3)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "-Kb1Q3eeP7DI" }, "source": [ "After 10 epochs we achieve 44, 64, and 72 % top 1, 3, and 5 hit rate. Not bad! But it can be better. " ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "2iQYi1kdSLtF" }, "outputs": [], "source": [ "correct_prediction = 0\n", "correct_top3 = 0\n", "correct_top5 = 0\n", "for ii in range(len(probs)):\n", " P_this = np.argmax(probs[ii])\n", " GT_this = test_generator.labels[ii]\n", " if P_this == GT_this:\n", " correct_prediction += 1\n", " correct_top3 += top3(probs[ii],GT_this)\n", " correct_top5 += top5(probs[ii],GT_this)\n", " if ii < 10:\n", " print(\"Prediction: {} ({})\".format(P_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(P_this)])]))\n", " print(\"Actual: {} ({})\".format(GT_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(GT_this)])]))\n", " print(\"Prediction: {} \".format(P_this))\n", " print(\"Actual: {} \".format(GT_this))\n", " #plt.figure()\n", " #plt.imshow(X_test[ii,:,:,:])\n", " #plt.show()\n", "print(correct_prediction, correct_prediction/len(probs))\n", "print(correct_top3, correct_top3/len(probs))\n", "print(correct_top5, correct_top5/len(probs))\n", "print(len(probs))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "2ou1-W3SQYOK" }, "source": [ "We see overfitting after second epoch, which seems to be a common thing when training models for bird IDs. " ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "7tJdIW8F6mIv" }, "outputs": [], "source": [ "acc = history.history['accuracy']\n", "val_acc = history.history['val_accuracy']\n", "\n", "loss = history.history['loss']\n", "val_loss = history.history['val_loss']\n", "\n", "plt.figure(figsize=(8, 8))\n", "plt.subplot(2, 1, 1)\n", "plt.plot(acc, label='Training Accuracy')\n", "plt.plot(val_acc, label='Validation Accuracy')\n", "plt.legend(loc='lower right')\n", "plt.ylabel('Accuracy')\n", "plt.ylim([min(plt.ylim()),1])\n", "plt.title('Training Accuracy')\n", "\n", "plt.subplot(2, 1, 2)\n", "plt.plot(loss, label='Training Loss')\n", "plt.plot(val_loss, label='Validation Loss')\n", "plt.legend(loc='upper right')\n", "plt.ylabel('Cross Entropy')\n", "# plt.ylim([0,1.0])\n", "plt.title('Training Loss')\n", "plt.xlabel('epoch')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "FK_OysQzQ4_S" }, "source": [ "We start to fine tune the model by allowing the last 55 layers to be trained." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "BffxJucn6wlf" }, "outputs": [], "source": [ "base_model.trainable = True" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "WCA4VLaUYr7P" }, "outputs": [], "source": [ "# Let's take a look to see how many layers are in the base model\n", "print(\"Number of layers in the base model: \", len(base_model.layers))\n", "\n", "# Fine tune from this layer onwards\n", "fine_tune_at = 100\n", "\n", "# Freeze all the layers before the `fine_tune_at` layer\n", "for layer in base_model.layers[:fine_tune_at]:\n", " layer.trainable = False" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "fbNXMyuEZd2h" }, "outputs": [], "source": [ "model.compile(loss='categorical_crossentropy',\n", " optimizer = tf.keras.optimizers.RMSprop(lr=base_learning_rate/10),\n", " metrics=['accuracy'])" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "JmYtpM-GZit-" }, "outputs": [], "source": [ "initial_epochs = 10\n", "fine_tune_epochs = 10\n", "total_epochs = initial_epochs + fine_tune_epochs\n", "\n", "\n", "def fit_model_FT(model, batch_size=32, epochs=10): \n", " history = model.fit_generator(\n", " generator=train_generator,\n", " steps_per_epoch=(len(train_frame) // batch_size),\n", " epochs=total_epochs,\n", " initial_epoch=initial_epochs,\n", " validation_data=validation_generator,\n", " callbacks=None\n", " )\n", " score = model.evaluate_generator(train_generator, verbose=1)\n", " probs = model.predict_generator(test_generator, verbose=1)\n", " return model, score, probs, history" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "Ng5tOm_VZ2LZ" }, "outputs": [], "source": [ "fit_model, score, probs, history = fit_model_FT(model, batch_size=32, epochs=10)" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "ee6U1HZLWnK8" }, "outputs": [], "source": [ "acc = history.history['accuracy']\n", "val_acc = history.history['val_accuracy']\n", "\n", "loss = history.history['loss']\n", "val_loss = history.history['val_loss']\n", "\n", "plt.figure(figsize=(8, 8))\n", "plt.subplot(2, 1, 1)\n", "plt.plot(acc, label='Training Accuracy')\n", "plt.plot(val_acc, label='Validation Accuracy')\n", "plt.legend(loc='lower right')\n", "plt.ylabel('Accuracy')\n", "plt.ylim([min(plt.ylim()),1])\n", "plt.title('Training Accuracy')\n", "\n", "plt.subplot(2, 1, 2)\n", "plt.plot(loss, label='Training Loss')\n", "plt.plot(val_loss, label='Validation Loss')\n", "plt.legend(loc='upper right')\n", "plt.ylabel('Cross Entropy')\n", "# plt.ylim([0,1.0])\n", "plt.title('Training Loss')\n", "plt.xlabel('epoch')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "bFU-cJgNRNr3" }, "source": [ "After 10 fine tuning epochs the hit rates increased to 63 (top), 82 (top3), and 88 % (top5)." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "fJWas9HFIm8p" }, "outputs": [], "source": [ "correct_prediction = 0\n", "correct_top3 = 0\n", "correct_top5 = 0\n", "for ii in range(len(probs)):\n", " P_this = np.argmax(probs[ii])\n", " GT_this = test_generator.labels[ii]\n", " if P_this == GT_this:\n", " correct_prediction += 1\n", " correct_top3 += top3(probs[ii],GT_this)\n", " correct_top5 += top5(probs[ii],GT_this)\n", " if ii % 100 == 20:\n", " t3, p3 = top3_idx(probs[ii])\n", " print(\"Prediction: {}, {}, or {} ({} ({:.1f} %), {} ({:.1f} %), or {} ({:.1f} %))\".format(t3[0], t3[1], t3[2] ,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[0])])], p3[0] * 100,\n", " Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[1])])], p3[1] * 100,\n", " Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[2])])], p3[2] * 100))\n", " print(\"Actual: {} ({})\".format(GT_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(GT_this)])]))\n", "\n", " \n", " plt.figure()\n", " image = plt.imread('data/test/'+test_generator.filenames[ii])\n", " plt.imshow(image)\n", " plt.show()\n", "print(correct_prediction, correct_prediction/len(probs))\n", "print(correct_top3, correct_top3/len(probs))\n", "print(correct_top5, correct_top5/len(probs))\n", "print(len(probs))" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "5BfjDfYfSaGs" }, "outputs": [], "source": [ "Prediction = []\n", "Prediction3 = []\n", "Correct_prediction3 = []\n", "for ii in range(len(probs)):\n", " Prediction.append(np.argmax(probs[ii]))\n", " Prediction3.append(top3_idx(probs[ii])[0])\n", " Correct_prediction3.append(np.asscalar(np.in1d(test_generator.labels[ii],Prediction3[ii])))\n", "\n", " \n", "Correct_predicted = []\n", "Correct_predicted3 = []\n", "Species_length = []\n", "for ii in range(len(np.unique(test_generator.labels))):\n", " Species_length.append(sum((test_generator.labels == ii)))\n", " Correct_predicted.append((sum((test_generator.labels == ii) & (Prediction == test_generator.labels)))/sum(test_generator.labels == ii))\n", " Correct_predicted3.append(sum((test_generator.labels == ii) & (Correct_prediction3))/sum(test_generator.labels == ii))\n" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "y090L25JReE_" }, "source": [ "For some species the model performed terribly. " ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "hYumabaNVOMA" }, "outputs": [], "source": [ "for ii in range(len(np.unique(test_generator.labels))):\n", "# if sum((test_generator.labels == ii)) < 8:\n", " if Correct_predicted3[ii] < 0.4:\n", " print('{}: {:.2f}, {:.2f}, {}, {}'.format(ii, Correct_predicted[ii], Correct_predicted3[ii], sum((train_generator.labels == ii)), Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(ii)])]))" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "eWALVyB-PwNz" }, "outputs": [], "source": [ "initial_epochs = 20\n", "fine_tune_epochs = 10\n", "total_epochs = initial_epochs + fine_tune_epochs\n", "\n", "\n", "def fit_model_FT2(model, batch_size=32, epochs=10): \n", " history = model.fit_generator(\n", " generator=train_generator,\n", " steps_per_epoch=(len(train_frame) // batch_size),\n", " epochs=total_epochs,\n", " initial_epoch=initial_epochs,\n", " validation_data=validation_generator,\n", " callbacks=None\n", " )\n", " score = model.evaluate_generator(train_generator, verbose=1)\n", " probs = model.predict_generator(test_generator, verbose=1)\n", " return model, score, probs, history" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "sqEGCIJmP3aC" }, "outputs": [], "source": [ "#fit_model, score, probs, history = fit_model_FT(model, X_train, X_test, Y_train, Y_test, batch_size=32, epochs=15)\n", "fit_model, score, probs, history = fit_model_FT2(model, batch_size=32, epochs=10)\n", "model.save(datapath + '/model3_30.h5')\n", "\n" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "4DyFi0YJH5iv" }, "outputs": [], "source": [ "probs = model.predict_generator(test_generator, verbose=1)\n", "np.savetxt(datapath + '/probs30.txt', probs)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "fB66sNewSaue" }, "source": [ "After another 10 fine tuning epochs (total = 30 epochs) we have 66, 84, 89 % top 1, 3, 5 hit rate. At this point it may be more helpful to refine the probability based on the location and time of year that the picture is taken. " ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "bjudiCZmKEe3" }, "outputs": [], "source": [ "correct_prediction = 0\n", "correct_top3 = 0\n", "correct_top5 = 0\n", "for ii in range(len(probs)):\n", " P_this = np.argmax(probs[ii])\n", " GT_this = test_generator.labels[ii]\n", " if P_this == GT_this:\n", " correct_prediction += 1\n", " correct_top3 += top3(probs[ii],GT_this)\n", " correct_top5 += top5(probs[ii],GT_this)\n", " if ii % 100 == 26:\n", "# if test_generator.labels[ii] == 10: # Gadwall\n", " #print(\"Prediction: {} ({})\".format(P_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(top3_idx(probs[ii]))])]))\n", " t3, p3 = top3_idx(probs[ii])\n", " print(\"Prediction: {}, {}, or {} ({} ({:.1f} %), {} ({:.1f} %), or {} ({:.1f} %))\".format(t3[0], t3[1], t3[2] ,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[0])])], p3[0] * 100,\n", " Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[1])])], p3[1] * 100,\n", " Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[2])])], p3[2] * 100))\n", " print(\"Actual: {} ({})\".format(GT_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(GT_this)])]))\n", "# print(\"Prediction: {} \".format(P_this))\n", "# print(\"Actual: {} \".format(GT_this))\n", "# print('data/test/'+test_generator.filenames[ii])\n", " \n", " plt.figure()\n", " image = plt.imread('data/test/'+test_generator.filenames[ii])\n", " plt.imshow(image)\n", " plt.show()\n", "print(correct_prediction, correct_prediction/len(probs))\n", "print(correct_top3, correct_top3/len(probs))\n", "print(correct_top5, correct_top5/len(probs))\n", "print(len(probs))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "RlxJGKb8TNYY" }, "source": [ "Using my photos to test." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "TRgQdjw7APzO" }, "outputs": [], "source": [ "Darren_test_generator = test_datagen.flow_from_directory(\n", " #directory=datapath + '/data/test/',\n", " directory='/content/Darren_test/',\n", " shuffle=False,\n", " #classes=list(Bird_list),\n", " target_size=(224, 224),\n", " batch_size=32,\n", " class_mode='categorical')" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "FbuTUb_6AaGy" }, "outputs": [], "source": [ "Darren_probs = model.predict_generator(Darren_test_generator, verbose=1)" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "bf_PABndAb7s" }, "outputs": [], "source": [ "correct_prediction = 0\n", "correct_top3 = 0\n", "correct_top5 = 0\n", "for ii in range(len(Darren_probs)):\n", "# for ii in [0]:\n", " P_this = np.argmax(Darren_probs[ii])\n", " GT_this = test_generator.class_indices[list(Darren_test_generator.class_indices.keys())[Darren_test_generator.labels[ii]]]\n", " if P_this == GT_this:\n", " correct_prediction += 1\n", " correct_top3 += top3(Darren_probs[ii],GT_this)\n", " correct_top5 += top5(Darren_probs[ii],GT_this)\n", "# if ii % 100 == 26:\n", "# if test_generator.labels[ii] == 10: # Gadwall\n", " #print(\"Prediction: {} ({})\".format(P_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(top3_idx(probs[ii]))])]))\n", " t3, p3 = top3_idx(Darren_probs[ii])\n", " print(\"Prediction: {}, {}, or {} ({} ({:.1f} %), {} ({:.1f} %), or {} ({:.1f} %))\".format(\n", " t3[0], t3[1], t3[2] ,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[0])])], p3[0] * 100,\n", " Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[1])])], p3[1] * 100,\n", " Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(t3[2])])], p3[2] * 100))\n", " print(\"Actual: {} ({})\".format(GT_this,Birds[int(list(test_generator.class_indices.keys())[list(test_generator.class_indices.values()).index(GT_this)])]))\n", "# print(\"Prediction: {} \".format(P_this))\n", "# print(\"Actual: {} \".format(GT_this))\n", "# print('data/test/'+test_generator.filenames[ii])\n", " \n", " plt.figure()\n", " image = plt.imread('Darren_test/'+Darren_test_generator.filenames[ii])\n", " plt.imshow(image)\n", " plt.show()\n", "print(correct_prediction, correct_prediction/len(Darren_probs))\n", "print(correct_top3, correct_top3/len(Darren_probs))\n", "print(correct_top5, correct_top5/len(Darren_probs))\n", "print(len(Darren_probs))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "Yg-w5GA6TIZy" }, "source": [ "Below are scratch codes." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "h_6UTR_-hBIs" }, "outputs": [], "source": [ "#Birds\n", "class_indices_inv_map = {v: k for k, v in test_generator.class_indices.items()} \n", "\n", "import pickle\n", "\n", "# write python dict to a file\n", "output = open(datapath+'/class_indices_inv_map.pkl', 'wb')\n", "pickle.dump(class_indices_inv_map, output)\n", "output.close()\n", "output = open(datapath+'/Birds.pkl', 'wb')\n", "pickle.dump(Birds, output)\n", "output.close()" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "Bird_ID_404_species_CNN_TL_clean.ipynb", "provenance": [], "version": "0.3.2" }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 1 }