Merge pull request #129 from egesko/master

Update notebook to work with new version of capsa

Author: aamini

lab3/Part2_BiasAndUncertainty.ipynb

@@ -40,6 +40,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "IgYKebt871EK"
@@ -59,6 +60,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "6JTRoM7E71EU"
@@ -95,6 +97,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "6VKVqLb371EV"
@@ -130,6 +133,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "cREmhMWJ71EX"
@@ -143,6 +147,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "1NhotGiT71EY"
@@ -199,6 +204,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "LgTG6buf71Ea"
@@ -256,6 +262,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "SzFGcrhv71Ed"
@@ -303,11 +310,12 @@
         "# Get all faces from the testing dataset\n",
         "test_imgs = test_loader.get_all_faces()\n",
         "\n",
-        "# Call the Capsa-wrapped classifier to generate outputs: predictions, uncertainty, and bias!\n",
-        "predictions, uncertainty, bias = wrapped_model.predict(test_imgs, batch_size=512)"
+        "# Call the Capsa-wrapped classifier to generate outputs: a RiskTensor dictionary consisting of predictions, uncertainty, and bias!\n",
+        "out = wrapped_model.predict(test_imgs, batch_size=512)\n"
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "629ng-_H6WOk"
@@ -329,10 +337,10 @@
         "### Analyzing representation bias scores ###\n",
         "\n",
         "# Sort according to lowest to highest representation scores\n",
-        "indices = np.argsort(bias, axis=None) # sort the score values themselves\n",
+        "indices = np.argsort(out.bias, axis=None) # sort the score values themselves\n",
         "sorted_images = test_imgs[indices] # sort images from lowest to highest representations\n",
-        "sorted_biases = bias[indices] # order the representation bias scores\n",
-        "sorted_preds = predictions[indices] # order the prediction values\n",
+        "sorted_biases = out.bias.numpy()[indices] # order the representation bias scores\n",
+        "sorted_preds = out.y_hat.numpy()[indices] # order the prediction values\n",
         "\n",
         "\n",
         "# Visualize the 20 images with the lowest and highest representation in the test dataset\n",
@@ -345,6 +353,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "-JYmGMJF71Ef"
@@ -368,6 +377,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "i8ERzg2-71Ef"
@@ -389,6 +399,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "cRNV-3SU71Eg"
@@ -404,6 +415,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "ww5lx7ue71Eg"
@@ -420,6 +432,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "NEfeWo2p7wKm"
@@ -442,10 +455,10 @@
         "### Analyzing epistemic uncertainty estimates ###\n",
         "\n",
         "# Sort according to epistemic uncertainty estimates\n",
-        "epistemic_indices = np.argsort(uncertainty, axis=None) # sort the uncertainty values\n",
+        "epistemic_indices = np.argsort(out.epistemic, axis=None) # sort the uncertainty values\n",
         "epistemic_images = test_imgs[epistemic_indices] # sort images from lowest to highest uncertainty\n",
-        "sorted_epistemic = uncertainty[epistemic_indices] # order the uncertainty scores\n",
-        "sorted_epistemic_preds = predictions[epistemic_indices] # order the prediction values\n",
+        "sorted_epistemic = out.epistemic.numpy()[epistemic_indices] # order the uncertainty scores\n",
+        "sorted_epistemic_preds = out.y_hat.numpy()[epistemic_indices] # order the prediction values\n",
         "\n",
         "\n",
         "# Visualize the 20 images with the LEAST and MOST epistemic uncertainty\n",
@@ -458,6 +471,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "L0dA8EyX71Eh"
@@ -481,6 +495,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "iyn0IE6x71Eh"
@@ -496,6 +511,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "XbwRbesM71Eh"
@@ -561,11 +577,11 @@
         "\n",
         "  # After the epoch is done, recompute data sampling proabilities \n",
         "  #  according to the inverse of the bias\n",
-        "  pred, unc, bias = wrapper(train_imgs)\n",
+        "  out = wrapper(train_imgs)\n",
         "\n",
         "  # Increase the probability of sampling under-represented datapoints by setting \n",
         "  #   the probability to the **inverse** of the biases\n",
-        "  inverse_bias = 1.0 / (bias.numpy() + 1e-7)\n",
+        "  inverse_bias = 1.0 / (np.mean(out.bias.numpy(),axis=-1) + 1e-7)\n",
         "\n",
         "  # Normalize the inverse biases in order to convert them to probabilities\n",
         "  p_faces = inverse_bias / np.sum(inverse_bias)\n",
@@ -575,6 +591,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "SwXrAeBo71Ej"
@@ -598,13 +615,13 @@
         "### Evaluation of debiased model ###\n",
         "\n",
         "# Get classification predictions, uncertainties, and representation bias scores\n",
-        "pred, unc, bias = wrapper.predict(test_imgs)\n",
+        "out = wrapper.predict(test_imgs)\n",
         "\n",
         "# Sort according to lowest to highest representation scores\n",
-        "indices = np.argsort(bias, axis=None)\n",
+        "indices = np.argsort(out.bias, axis=None)\n",
         "bias_images = test_imgs[indices] # sort the images\n",
-        "sorted_bias = bias[indices] # sort the representation bias scores\n",
-        "sorted_bias_preds = pred[indices] # sort the predictions\n",
+        "sorted_bias = out.bias.numpy()[indices] # sort the representation bias scores\n",
+        "sorted_bias_preds = out.y_hat.numpy()[indices] # sort the predictions\n",
         "\n",
         "# Plot the representation bias vs. the accuracy\n",
         "plt.xlabel(\"Density (Representation)\")\n",
@@ -613,6 +630,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "d1cEEnII71Ej"
@@ -681,7 +699,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.9.6 (default, Oct 18 2022, 12:41:40) \n[Clang 14.0.0 (clang-1400.0.29.202)]"
+      "version": "3.9.16"
     },
     "vscode": {
       "interpreter": {