{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# One bit, 9,994 birds\n",
        "\n",
        "This notebook reproduces the exhaustive final-layer bit-flip experiment behind **I Flipped One Bit in a Neural Network Until It Developed a Favorite Animal**. It evaluates one pinned CIFAR-10 ResNet-20 checkpoint; it does not claim that every neural network has the same failure."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from pathlib import Path\n",
        "import os\n",
        "import subprocess\n",
        "\n",
        "local = Path.cwd()\n",
        "if (local / 'run_experiment.py').exists():\n",
        "    experiment = local\n",
        "else:\n",
        "    repo = Path('/content/jasonsuhari.com')\n",
        "    if not repo.exists():\n",
        "        subprocess.run(['git', 'clone', '--depth', '1', 'https://github.com/jasonsuhari/jasonsuhari.com.git', str(repo)], check=True)\n",
        "    experiment = repo / 'public' / 'research' / 'neural-network-bit-flip'\n",
        "\n",
        "if not (experiment / 'run_experiment.py').exists():\n",
        "    raise FileNotFoundError('Upload this notebook beside run_experiment.py, or clone a revision that includes the published experiment.')\n",
        "os.chdir(experiment)\n",
        "print(experiment)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "import sys\n",
        "subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt'], check=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Run the experiment\n",
        "\n",
        "The first run downloads the checksum-verified CIFAR-10 archive and pinned model checkpoint. A GPU run is normally under a minute after downloads; CPU runtime depends heavily on the Colab host."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "import torch\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "subprocess.run([sys.executable, '-u', 'run_experiment.py', '--device', device], check=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "subprocess.run([sys.executable, 'verify_results.py'], check=True)\n",
        "\n",
        "import json\n",
        "from IPython.display import Image, display\n",
        "results = json.loads(Path('outputs/results.json').read_text())\n",
        "winner = results['winners_full_test']['float32']\n",
        "print({\n",
        "    'baseline_accuracy': results['baseline_float32_full_test']['accuracy'],\n",
        "    'favorite_animal': winner['target_class'],\n",
        "    'mutated_accuracy': winner['full_test_accuracy'],\n",
        "    'favorite_prediction_share': winner['full_test_target_share'],\n",
        "    'old_bits': winner['old_bits_hex'],\n",
        "    'new_bits': winner['new_bits_hex'],\n",
        "})\n",
        "display(Image(filename='outputs/prediction-collapse-evidence.png'))"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "one-bit-bird-colab.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}