{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 代码实现"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来，我们将动手编写实现基于视觉词袋模型的图像分类算法。我们先导入必要的库，并导入Caltech-101数据集。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-03T19:00:39.189642Z",
     "iopub.status.busy": "2024-12-03T19:00:39.189303Z",
     "iopub.status.idle": "2024-12-03T19:00:49.344519Z",
     "shell.execute_reply": "2024-12-03T19:00:49.342657Z",
     "shell.execute_reply.started": "2024-12-03T19:00:39.189607Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import cv2\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.kernel_approximation import AdditiveChi2Sampler\n",
    "import random\n",
    "import math\n",
    "from imutils import paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-03T19:00:56.750681Z",
     "iopub.status.busy": "2024-12-03T19:00:56.750216Z",
     "iopub.status.idle": "2024-12-03T19:01:32.317113Z",
     "shell.execute_reply": "2024-12-03T19:01:32.315884Z",
     "shell.execute_reply.started": "2024-12-03T19:00:56.750637Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# 图像数据\n",
    "data = []\n",
    "# 图像对应的标签\n",
    "labels = []\n",
    "# 储存标签信息的临时变量\n",
    "labels_tep = []\n",
    "# 数据集的地址\n",
    "# 请读者自行下载数据集，并将数据集放在代码的同级目录下\n",
    "image_paths = list(paths.list_images('./caltech-101'))\n",
    "\n",
    "for image_path in image_paths:\n",
    "    # 获取图像类别\n",
    "    label = image_path.split(os.path.sep)[-2]\n",
    "    # 读取每个类别的图像\n",
    "    image = cv2.imread(image_path)\n",
    "    # 将图像通道从BGR转换为RGB\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "    # 统一输入图像的尺寸\n",
    "    image = cv2.resize(image, (200,200), interpolation=cv2.INTER_AREA)\n",
    "    data.append(image)\n",
    "    labels_tep.append(label)\n",
    "\n",
    "name2label = {}\n",
    "tep = {}\n",
    "for idx, name in enumerate(labels_tep):\n",
    "    tep[name] = idx\n",
    "for idx, name in enumerate(tep):\n",
    "    name2label[name] = idx\n",
    "for idx, image_path in enumerate(image_paths):\n",
    "    labels.append(name2label[image_path.split(os.path.sep)[-2]])\n",
    "\n",
    "data = np.array(data)\n",
    "labels = np.array(labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们接着对数据集进行训练集验证集和测试集的划分。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-03T19:01:43.294246Z",
     "iopub.status.busy": "2024-12-03T19:01:43.293809Z",
     "iopub.status.idle": "2024-12-03T19:01:43.898497Z",
     "shell.execute_reply": "2024-12-03T19:01:43.897052Z",
     "shell.execute_reply.started": "2024-12-03T19:01:43.294212Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train examples: (5486, 200, 200, 3)\n",
      "      x_test examples: (1829, 200, 200, 3)\n",
      "        x_val examples: (1829, 200, 200, 3)\n"
     ]
    }
   ],
   "source": [
    "(x_train, X, y_train, Y) = train_test_split(data, labels, \n",
    "        test_size=0.4, stratify=labels, random_state=42)\n",
    "(x_val, x_test, y_val, y_test) = train_test_split(X, Y, \n",
    "        test_size=0.5, random_state=42)\n",
    "print(f\"x_train examples: {x_train.shape}\\n\\\n",
    "      x_test examples: {x_test.shape}\\n\\\n",
    "        x_val examples: {x_val.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-14T04:15:42.707808Z",
     "iopub.status.busy": "2022-11-14T04:15:42.706668Z",
     "iopub.status.idle": "2022-11-14T04:15:42.717775Z",
     "shell.execute_reply": "2022-11-14T04:15:42.716426Z",
     "shell.execute_reply.started": "2022-11-14T04:15:42.707764Z"
    }
   },
   "source": [
    "我们先进行视觉词典的构建。为了得到视觉词典，首先需要对输入图像的表征进行提取，这里使用SIFT对图像进行处理。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-03T19:01:46.199836Z",
     "iopub.status.busy": "2024-12-03T19:01:46.199431Z",
     "iopub.status.idle": "2024-12-03T19:03:15.830468Z",
     "shell.execute_reply": "2024-12-03T19:03:15.829162Z",
     "shell.execute_reply.started": "2024-12-03T19:01:46.199800Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# 构建一个词典，储存每一个类别的sift信息\n",
    "vec_dict = {i:{'kp':[], 'des':{}} for i in range(102)}\n",
    "\n",
    "sift = cv2.SIFT_create()\n",
    "for i in range(x_train.shape[0]):\n",
    "    # 对图像正则化\n",
    "    tep = cv2.normalize(x_train[i], None, 0, 255, \n",
    "                        cv2.NORM_MINMAX).astype('uint8')\n",
    "    # 计算图像的SIFT特征\n",
    "    kp_vector, des_vector = sift.detectAndCompute(tep, None)\n",
    "    # 特征点和描述符信息储存进词典中\n",
    "    vec_dict[y_train[i]]['kp'] += list(kp_vector)\n",
    "    for k in range(len(kp_vector)):\n",
    "        # des使用kp_vector将其一一对应\n",
    "        vec_dict[y_train[i]]['des'][kp_vector[k]] = des_vector[k]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接着，为了保持类别之间的平衡，为每个类别选取相同个数的特征用于之后的聚类。先统计每个类别的SIFT图像特征个数，找到拥有最少特征个数的类别，然后以该类别的特征个数为标准个数，从每个类别的特征中筛选出标准个数的特征进行后续的聚类任务。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-03T19:06:49.636848Z",
     "iopub.status.busy": "2024-12-03T19:06:49.635984Z",
     "iopub.status.idle": "2024-12-03T19:06:50.591340Z",
     "shell.execute_reply": "2024-12-03T19:06:50.590130Z",
     "shell.execute_reply.started": "2024-12-03T19:06:49.636798Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# 设置最少特征点的数目\n",
    "bneck_value = float(\"inf\")\n",
    "\n",
    "# 确定bneck_value的数值\n",
    "for i in range(102):\n",
    "    if len(vec_dict[i]['kp']) < bneck_value:\n",
    "        bneck_value = len(vec_dict[i]['kp'])\n",
    "\n",
    "# 按照每一个SIFT特征点的响应值对特征进行降序排序\n",
    "for i in range(102):\n",
    "    kp_list = vec_dict[i]['kp'] = sorted((vec_dict[i]['kp']), \n",
    "                                         key=lambda x: x.response, \n",
    "                                         reverse=True)\n",
    "    \n",
    "    \n",
    "        \n",
    "# 为每个类别选择同样多的特征点用于聚类。特征点个数bneck_value\n",
    "vec_list=[]\n",
    "for i in range(bneck_value):\n",
    "    vec_list.append(vec_dict[0]['des'][vec_dict[0]['kp'][i]])\n",
    "\n",
    "for i in range(1, 102):\n",
    "    for j in range(bneck_value):\n",
    "        vec_list.append(vec_dict[i]['des'][vec_dict[i]['kp'][j]])\n",
    "\n",
    "vec_list = np.float64(vec_list)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "然后，使用$k$-means聚类算法将这些特征进行聚类，得到的聚类中心组成视觉词典。这里，将聚类中心个数设置为200。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-03T19:07:41.340947Z",
     "iopub.status.busy": "2024-12-03T19:07:41.340507Z",
     "iopub.status.idle": "2024-12-03T19:21:49.036470Z",
     "shell.execute_reply": "2024-12-03T19:21:49.035082Z",
     "shell.execute_reply.started": "2024-12-03T19:07:41.340911Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "N_clusters = 200\n",
    "kmeans = KMeans(n_clusters=N_clusters, random_state=0).fit(vec_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们引入空间金字塔匹配算法提升词袋模型的表征能力。\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-03T19:22:15.145246Z",
     "iopub.status.busy": "2024-12-03T19:22:15.144834Z",
     "iopub.status.idle": "2024-12-03T19:22:15.158571Z",
     "shell.execute_reply": "2024-12-03T19:22:15.157175Z",
     "shell.execute_reply.started": "2024-12-03T19:22:15.145213Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "def extract_SIFT(img):\n",
    "    sift = cv2.SIFT_create()\n",
    "    descriptors = []\n",
    "    for disft_step_size in DSIFT_STEP_SIZE:\n",
    "        keypoints = [cv2.KeyPoint(x, y, disft_step_size)\n",
    "                for y in range(0, img.shape[0], disft_step_size)\n",
    "                    for x in range(0, img.shape[1], disft_step_size)]\n",
    "\n",
    "        descriptors.append(sift.compute(img, keypoints)[1])\n",
    "    \n",
    "    return np.concatenate(descriptors, axis=0).astype('float64')\n",
    "\n",
    "\n",
    "# 获取图像的SPM特征\n",
    "def getImageFeaturesSPM(L, img, kmeans, k):\n",
    "    W = img.shape[1]\n",
    "    H = img.shape[0]   \n",
    "    h = []\n",
    "    for l in range(L+1):\n",
    "        w_step = math.floor(W/(2**l))\n",
    "        h_step = math.floor(H/(2**l))\n",
    "        x, y = 0, 0\n",
    "        for _ in range(2**l):\n",
    "            x = 0\n",
    "            for __ in range(2**l):\n",
    "                desc = extract_SIFT(img[y:y+h_step, x:x+w_step])                \n",
    "                predict = kmeans.predict(desc)\n",
    "                histo = np.bincount(predict, \n",
    "                        minlength=k).reshape(1,-1).ravel()\n",
    "                weight = 2**(l-L)\n",
    "                h.append(weight*histo)\n",
    "                x = x + w_step\n",
    "            y = y + h_step\n",
    "            \n",
    "    hist = np.array(h).ravel()\n",
    "    hist /= np.sum(hist)\n",
    "    return hist\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "分别使用SPM提取训练集和测试集图像的特征。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-03T19:22:23.283823Z",
     "iopub.status.busy": "2024-12-03T19:22:23.283424Z",
     "iopub.status.idle": "2024-12-03T19:52:27.955380Z",
     "shell.execute_reply": "2024-12-03T19:52:27.953739Z",
     "shell.execute_reply.started": "2024-12-03T19:22:23.283791Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# SPM 的一些超参数\n",
    "pyramid_level = 2\n",
    "DSIFT_STEP_SIZE = [4, 8]\n",
    "\n",
    "hist_vector = []\n",
    "for i in range(x_train.shape[0]):\n",
    "    tep = cv2.normalize(x_train[i], None, 0, 255, cv2.NORM_MINMAX).astype('uint8')\n",
    "    # 提取图像的SPM特征\n",
    "    hist_SPM = getImageFeaturesSPM(pyramid_level, tep, kmeans, N_clusters)\n",
    "    # 将提取的特征加入直方图中\n",
    "    hist_vector.append(hist_SPM)\n",
    "hist_vector = np.array(hist_vector)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-03T19:53:01.893970Z",
     "iopub.status.busy": "2024-12-03T19:53:01.893469Z",
     "iopub.status.idle": "2024-12-03T20:02:59.354768Z",
     "shell.execute_reply": "2024-12-03T20:02:59.353272Z",
     "shell.execute_reply.started": "2024-12-03T19:53:01.893926Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "hist_test_vector = []\n",
    "for i in range(x_test.shape[0]):\n",
    "    tep = cv2.normalize(x_test[i], None, 0, 255, cv2.NORM_MINMAX).astype('uint8')\n",
    "    hist_SPM = getImageFeaturesSPM(pyramid_level, tep, kmeans, N_clusters)\n",
    "    hist_test_vector.append(hist_SPM)\n",
    "hist_test_vector = np.array(hist_test_vector)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最后，训练一个SVM分类器，进行图像分类。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-03T20:08:38.781170Z",
     "iopub.status.busy": "2024-12-03T20:08:38.780723Z",
     "iopub.status.idle": "2024-12-03T20:09:56.283098Z",
     "shell.execute_reply": "2024-12-03T20:09:56.281891Z",
     "shell.execute_reply.started": "2024-12-03T20:08:38.781118Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC()"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "\n",
    "# transform使用一种特征映射的方法\n",
    "transformer = AdditiveChi2Sampler()\n",
    "transformer = transformer.fit(np.concatenate(\n",
    "    [hist_vector, hist_test_vector], axis=0)\n",
    "                             )\n",
    "\n",
    "# 构建SVM分类器\n",
    "classifier = svm.LinearSVC()\n",
    "# 将训练的直方图进行特征映射\n",
    "hist_vector = transformer.transform(hist_vector)\n",
    "\n",
    "# 对数据进行拟合\n",
    "classifier.fit(hist_vector, y_train)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最后，在测试集上对分类器进行评估，并计算 top-1 错误率。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-03T20:10:21.193007Z",
     "iopub.status.busy": "2024-12-03T20:10:21.192558Z",
     "iopub.status.idle": "2024-12-03T20:10:21.845889Z",
     "shell.execute_reply": "2024-12-03T20:10:21.844161Z",
     "shell.execute_reply.started": "2024-12-03T20:10:21.192974Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "top-1 error:  0.3001640240568617\n"
     ]
    }
   ],
   "source": [
    "# 将测试的直方图进行特征映射\n",
    "hist_test_vector = transformer.transform(hist_test_vector)\n",
    "\n",
    "# 计算分类 top-1 错误率\n",
    "top1_error = classifier.predict(hist_test_vector) - y_test\n",
    "tep = len(top1_error[top1_error!=0])\n",
    "print('top-1 error: ', tep/len(y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "为了更深入的了解ResNet，我们先动手编写ResNet34网络。\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "import torch as t\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "\n",
    "# 残差结构\n",
    "class ResidualBlock(nn.Module):\n",
    "    \n",
    "    # 深度学习中的图像和利用模型提取的特征图往往有很多通道（channel）\n",
    "    # 比如RGB图像的通道为3，即R通道、B通道与G通道\n",
    "    def __init__(self, inchannel, outchannel, stride=1, shortcut=None):\n",
    "        super(ResidualBlock, self).__init__()\n",
    "        # 观察图 10-6，不难发现残差结构可大致分为左右两部分\n",
    "        # 左边是一系列的网络层级，右边是一个跳跃连接\n",
    "        # 定义左边\n",
    "        self.left = nn.Sequential(\n",
    "            # 对应图 10-6 中第一个权重层\n",
    "            nn.Conv2d(inchannel, outchannel, 3, stride, 1, bias=False),\n",
    "            nn.BatchNorm2d(outchannel),\n",
    "            \n",
    "            # 对应图 10-6 中的 RELU\n",
    "            nn.ReLU(inplace=True),\n",
    "            \n",
    "            # 对应图 10-6 中第二个权重层\n",
    "            nn.Conv2d(outchannel, outchannel, 3, 1, 1, bias=False),\n",
    "            nn.BatchNorm2d(outchannel))\n",
    "        \n",
    "        # 定义右边\n",
    "        self.right = shortcut\n",
    "        \n",
    "    # forward()函数在网络结构中起到举足轻重的作用，它决定着网络如何对数据进行传播\n",
    "    def forward(self, x):\n",
    "        out = self.left(x)\n",
    "        # 构建残差结构\n",
    "        residual = x if self.right is None else self.right(x)\n",
    "        out += residual\n",
    "        \n",
    "        return F.relu(out)\n",
    "\n",
    "\n",
    "# 在这一模块中，将实现 ResNet34，其对应的结构可查看图 10-7\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, num_classes=102):\n",
    "        super(ResNet, self).__init__()\n",
    "        # 前几层图像转换\n",
    "        self.pre = nn.Sequential(\n",
    "            nn.Conv2d(3, 64, 7, 2, 3, bias=False),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(3, 2, 1),\n",
    "        )\n",
    "\n",
    "        # 重复的网络层分别有3、4、6、3个残差块\n",
    "        self.layer1 = self._make_layer(64, 128, 3)\n",
    "        self.layer2 = self._make_layer(128, 256, 4, stride=2)\n",
    "        self.layer3 = self._make_layer(256, 512, 6, stride=2)\n",
    "        self.layer4 = self._make_layer(512, 512, 3, stride=2)\n",
    "\n",
    "        # 分类用的全连接层，将一个多通道的特征图映射到一个维度为类别数目的向量\n",
    "        self.fc = nn.Linear(512, num_classes)\n",
    "\n",
    "    def _make_layer(self, inchannel, outchannel, block_num, stride=1):\n",
    "        # 定义shortcut连接\n",
    "        shortcut = nn.Sequential(\n",
    "            nn.Conv2d(inchannel, outchannel, 1, stride, bias=False),\n",
    "            nn.BatchNorm2d(outchannel))\n",
    "\n",
    "        layers = []\n",
    "        \n",
    "        # 给当前网络层级添加残差块\n",
    "        layers.append(ResidualBlock(inchannel, outchannel, \n",
    "                                    stride, shortcut))\n",
    "\n",
    "        for i in range(1, block_num):\n",
    "            layers.append(ResidualBlock(outchannel, outchannel))\n",
    "\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pre(x)\n",
    "\n",
    "        x = self.layer1(x)\n",
    "        x = self.layer2(x)\n",
    "        x = self.layer3(x)\n",
    "        x = self.layer4(x)\n",
    "\n",
    "        x = F.avg_pool2d(x, 7)\n",
    "        x = x.view(x.size(0), -1)\n",
    "\n",
    "        return self.fc(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以使用nn.Covn2d()函数实现卷积操作，该函数定义如下："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "nn.Conv2d(in_channels,          # 输入的通道数\n",
    "          out_channels,         # 输出的通道数\n",
    "          kernel_size,          # 卷积核的大小\n",
    "          stride=1,             # 卷积核移动的步长\n",
    "          padding=0,            # 卷积核补零的层数\n",
    "          )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来，我们将动手编写完成基于Resnet34的图像分类任务。首先，先导入必要的库。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "import torch\n",
    "import torch.optim\n",
    "import torch.utils.data as Data\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "import torchvision.models\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们对数据进行增强。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# 定义训练图像增强（变换）的方法\n",
    "train_transform = transforms.Compose(\n",
    "    [transforms.ToPILImage(),\n",
    "     # transforms.Resize((224, 224)),\n",
    "     transforms.ToTensor(),\n",
    "     transforms.Normalize(mean=[0.485, 0.456, 0.406], \n",
    "                          std=[0.229, 0.224, 0.225])])\n",
    "\n",
    "# 定义训练图像增强（变换）的方法\n",
    "val_transform = transforms.Compose(\n",
    "    [transforms.ToPILImage(),\n",
    "     # transforms.Resize((224, 224)),\n",
    "     transforms.ToTensor(),\n",
    "     transforms.Normalize(mean=[0.485, 0.456, 0.406], \n",
    "                          std=[0.229, 0.224, 0.225])])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接着，把Caltech-101数据集封装成类，并生成训练集类、验证集类以及测试集类。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# 数据集\n",
    "class ImageDataset(Dataset):\n",
    "    def __init__(self, images, labels=None, transforms=None):\n",
    "        self.X = images\n",
    "        self.y = labels\n",
    "        self.transforms = transforms\n",
    "\n",
    "    def __len__(self):\n",
    "        return (len(self.X))\n",
    "    \n",
    "    # 用于深度学习训练构建的数据类中，__getitem__()函数非常重要\n",
    "    # __getitem__()决定着数据如何传入模型\n",
    "    # 在下面的代码中，可以发现，当transforms非空时:\n",
    "    # 数据将先经过transforms进行数据增强，再返回进行后续操作\n",
    "    def __getitem__(self, i):\n",
    "        data = self.X[i][:]\n",
    "\n",
    "        if self.transforms:\n",
    "            data = self.transforms(data)\n",
    "\n",
    "        if self.y is not None:\n",
    "            return (data, self.y[i])\n",
    "        else:\n",
    "            return data\n",
    "\n",
    "# 生成不同的类用于训练、验证以及测试\n",
    "train_data = ImageDataset(x_train, y_train, train_transform)\n",
    "val_data = ImageDataset(x_val, y_val, val_transform)\n",
    "test_data = ImageDataset(x_test, y_test, val_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 2048\n",
    "\n",
    "trainloader = DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True)\n",
    "valloader = DataLoader(val_data, batch_size=BATCH_SIZE, shuffle=True)\n",
    "testloader = DataLoader(test_data, batch_size=BATCH_SIZE, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来，我们开始加载ResNet34模型。这里将直接调用已经封装好的库。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# 根据当前设备选择使用CPU或者GPU训练\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "\n",
    "# 加载ImageNet预训练的模型\n",
    "model = torchvision.models.resnet34(pretrained='imagenet')\n",
    "num_ftrs = model.fc.in_features\n",
    "model.fc = nn.Linear(num_ftrs, 102)\n",
    "model.to(device)\n",
    "\n",
    "# 多张GPU并行训练\n",
    "model = nn.DataParallel(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "继续定义训练时需要用到的优化器和损失函数："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# 定义优化器\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-4)\n",
    "# 定义损失函数\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来，编写训练模型、验证模型以及测试模型top-1错误率的函数。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# 定义训练函数\n",
    "def fit(model, dataloader):\n",
    "\n",
    "    model.train()\n",
    "    \n",
    "    # 初始化模型损失以及模型 top-1 错误率\n",
    "    running_loss = 0.0\n",
    "    running_top1_error = 0\n",
    "    \n",
    "    # 开始迭代\n",
    "    for i, data in enumerate(dataloader):\n",
    "        \n",
    "        # 准备好训练的图像和标签，每次传入的数据量为batch_size\n",
    "        x, y = data[0].to(device), data[1].to(device)\n",
    "        # 需要在开始进行反向传播之前将梯度设置为零\n",
    "        # 因为PyTorch会在随后的反向传递中累积梯度\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # 将数据传入模型中，获得输出的预测标签\n",
    "        outputs = model(x)\n",
    "        \n",
    "        # 将预测标签与真实标签计算损失\n",
    "        loss = criterion(outputs, y)\n",
    "        \n",
    "        # 记录当前损失\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # 记录当前 top-1 错误率\n",
    "        _, preds = torch.max(outputs.data, 1)\n",
    "        running_top1_error += torch.sum(preds != y)\n",
    "        \n",
    "        # 反向传播\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "\n",
    "    loss = running_loss / len(dataloader.dataset)\n",
    "    top1_error = running_top1_error / len(dataloader.dataset)\n",
    "\n",
    "    print(f\"Train Loss: {loss:.4f}, Train Top-1 Error: {top1_error:.4f}\")\n",
    "\n",
    "    return loss, top1_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# 定义验证函数，可以通过验证集对模型的参数进行调整\n",
    "def validate(model, dataloader):\n",
    "\n",
    "    model.eval()\n",
    "    \n",
    "    # 初始化模型损失以及模型 top-1 错误率\n",
    "    running_loss = 0.0\n",
    "    running_top1_error = 0\n",
    "    with torch.no_grad():\n",
    "        for i, data in enumerate(dataloader):\n",
    "            # 流程同训练\n",
    "            x, y = data[0].to(device), data[1].to(device)\n",
    "            outputs = model(x)\n",
    "            loss = criterion(outputs, y)\n",
    "\n",
    "            running_loss += loss.item()\n",
    "            _, preds = torch.max(outputs.data, 1)\n",
    "            running_top1_error += torch.sum(preds != y)\n",
    "            \n",
    "        loss = running_loss / len(dataloader.dataset)\n",
    "        top1_error = running_top1_error / len(dataloader.dataset)\n",
    "        print(f'Val Loss: {loss:.4f}, Val Top-1 Error: {top1_error:.4f}')\n",
    "\n",
    "        return loss, top1_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# 定义测试函数，评估模型的效果\n",
    "def test(model, dataloader):\n",
    "    error = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for data in dataloader:\n",
    "            x, y = data[0].to(device), data[1].to(device)\n",
    "            outputs = model(x)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            total += y.size(0)\n",
    "            error += torch.sum(predicted != y)\n",
    "    return error, total\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "开始训练模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training on 5486 examples,       validating on 1829 examples...\n",
      "\n",
      "Epoch 1 of 25\n",
      "Train Loss: 0.0024, Train Top-1 Error: 0.8977\n",
      "Val Loss: 0.0019, Val Top-1 Error: 0.6490\n",
      "\n",
      "Epoch 2 of 25\n",
      "Train Loss: 0.0017, Train Top-1 Error: 0.5859\n",
      "Val Loss: 0.0015, Val Top-1 Error: 0.5189\n",
      "\n",
      "Epoch 3 of 25\n",
      "Train Loss: 0.0013, Train Top-1 Error: 0.4424\n",
      "Val Loss: 0.0012, Val Top-1 Error: 0.4117\n",
      "\n",
      "Epoch 4 of 25\n",
      "Train Loss: 0.0011, Train Top-1 Error: 0.3208\n",
      "Val Loss: 0.0010, Val Top-1 Error: 0.3461\n",
      "\n",
      "Epoch 5 of 25\n",
      "Train Loss: 0.0009, Train Top-1 Error: 0.2415\n",
      "Val Loss: 0.0009, Val Top-1 Error: 0.2592\n",
      "\n",
      "Epoch 6 of 25\n",
      "Train Loss: 0.0007, Train Top-1 Error: 0.1743\n",
      "Val Loss: 0.0007, Val Top-1 Error: 0.2056\n",
      "\n",
      "Epoch 7 of 25\n",
      "Train Loss: 0.0005, Train Top-1 Error: 0.1185\n",
      "Val Loss: 0.0006, Val Top-1 Error: 0.1651\n",
      "\n",
      "Epoch 8 of 25\n",
      "Train Loss: 0.0004, Train Top-1 Error: 0.0849\n",
      "Val Loss: 0.0005, Val Top-1 Error: 0.1405\n",
      "\n",
      "Epoch 9 of 25\n",
      "Train Loss: 0.0003, Train Top-1 Error: 0.0631\n",
      "Val Loss: 0.0004, Val Top-1 Error: 0.1241\n",
      "\n",
      "Epoch 10 of 25\n",
      "Train Loss: 0.0003, Train Top-1 Error: 0.0450\n",
      "Val Loss: 0.0004, Val Top-1 Error: 0.1061\n",
      "\n",
      "Epoch 11 of 25\n",
      "Train Loss: 0.0002, Train Top-1 Error: 0.0270\n",
      "Val Loss: 0.0003, Val Top-1 Error: 0.0973\n",
      "\n",
      "Epoch 12 of 25\n",
      "Train Loss: 0.0001, Train Top-1 Error: 0.0149\n",
      "Val Loss: 0.0003, Val Top-1 Error: 0.0869\n",
      "\n",
      "Epoch 13 of 25\n",
      "Train Loss: 0.0001, Train Top-1 Error: 0.0066\n",
      "Val Loss: 0.0003, Val Top-1 Error: 0.0809\n",
      "\n",
      "Epoch 14 of 25\n",
      "Train Loss: 0.0001, Train Top-1 Error: 0.0042\n",
      "Val Loss: 0.0003, Val Top-1 Error: 0.0727\n",
      "\n",
      "Epoch 15 of 25\n",
      "Train Loss: 0.0001, Train Top-1 Error: 0.0018\n",
      "Val Loss: 0.0002, Val Top-1 Error: 0.0705\n",
      "\n",
      "Epoch 16 of 25\n",
      "Train Loss: 0.0000, Train Top-1 Error: 0.0009\n",
      "Val Loss: 0.0002, Val Top-1 Error: 0.0694\n",
      "\n",
      "Epoch 17 of 25\n",
      "Train Loss: 0.0000, Train Top-1 Error: 0.0009\n",
      "Val Loss: 0.0002, Val Top-1 Error: 0.0678\n",
      "\n",
      "Epoch 18 of 25\n",
      "Train Loss: 0.0000, Train Top-1 Error: 0.0004\n",
      "Val Loss: 0.0002, Val Top-1 Error: 0.0683\n",
      "\n",
      "Epoch 19 of 25\n",
      "Train Loss: 0.0000, Train Top-1 Error: 0.0004\n",
      "Val Loss: 0.0002, Val Top-1 Error: 0.0694\n",
      "\n",
      "Epoch 20 of 25\n",
      "Train Loss: 0.0000, Train Top-1 Error: 0.0005\n",
      "Val Loss: 0.0002, Val Top-1 Error: 0.0667\n",
      "\n",
      "Epoch 21 of 25\n",
      "Train Loss: 0.0000, Train Top-1 Error: 0.0002\n",
      "Val Loss: 0.0002, Val Top-1 Error: 0.0672\n",
      "\n",
      "Epoch 22 of 25\n",
      "Train Loss: 0.0000, Train Top-1 Error: 0.0002\n",
      "Val Loss: 0.0002, Val Top-1 Error: 0.0672\n",
      "\n",
      "Epoch 23 of 25\n",
      "Train Loss: 0.0000, Train Top-1 Error: 0.0002\n",
      "Val Loss: 0.0002, Val Top-1 Error: 0.0683\n",
      "\n",
      "Epoch 24 of 25\n",
      "Train Loss: 0.0000, Train Top-1 Error: 0.0004\n",
      "Val Loss: 0.0002, Val Top-1 Error: 0.0683\n",
      "\n",
      "Epoch 25 of 25\n",
      "Train Loss: 0.0000, Train Top-1 Error: 0.0002\n",
      "Val Loss: 0.0002, Val Top-1 Error: 0.0667\n"
     ]
    }
   ],
   "source": [
    "# 开始对模型进行训练\n",
    "\n",
    "# 设定迭代的轮次\n",
    "epochs = 25\n",
    "# 设定训练以及验证的损失与 top-1 错误率\n",
    "train_loss , train_top1_error = [], []\n",
    "val_loss , val_top1_error = [], []\n",
    "\n",
    "print(f\"Training on {len(train_data)} examples, \\\n",
    "      validating on {len(val_data)} examples...\")\n",
    "\n",
    "# 开始迭代\n",
    "for epoch in range(epochs):\n",
    "    # 输出迭代信息\n",
    "    print(f\"\\nEpoch {epoch+1} of {epochs}\")\n",
    "    train_epoch_loss, train_epoch_top1_error = fit(model, trainloader)\n",
    "    val_epoch_loss, val_epoch_top1_error = validate(model, valloader)\n",
    "    train_loss.append(train_epoch_loss)\n",
    "    train_top1_error.append(train_epoch_top1_error.cpu())\n",
    "    val_loss.append(val_epoch_loss)\n",
    "    val_top1_error.append(val_epoch_top1_error.cpu())\n",
    "    \n",
    "# 可以保存训练的模型\n",
    "# torch.save(model.state_dict(), model.pth\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "绘制并观察 top-1 错误率和损失曲线。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fd1fd5df880>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 39944 characters]",
      "text/plain": [
       "<Figure size 1000x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 40960 characters]",
      "text/plain": [
       "<Figure size 1000x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 绘制 top-1 错误率曲线\n",
    "plt.figure(figsize=(10, 7))\n",
    "# 训练集的top-1 错误率曲线\n",
    "plt.plot(train_top1_error, color='green', label='train top-1 error')\n",
    "# 验证集top-1 错误率曲线\n",
    "plt.plot(val_top1_error, color='blue', label='validataion top-1 error')\n",
    "plt.xlabel('迭代次数')\n",
    "plt.ylabel('top-1 错误率')\n",
    "plt.legend()\n",
    "\n",
    "# 绘制损失函数曲线\n",
    "plt.figure(figsize=(10, 7))\n",
    "plt.plot(train_loss, color='orange', label='train loss')\n",
    "plt.plot(val_loss, color='red', label='validataion loss')\n",
    "plt.xlabel('迭代次数')\n",
    "plt.ylabel('损失函数')\n",
    "plt.legend()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最后，对模型的 top-1 错误率进行评估。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top-1 error of the network on test images: 5.358 %\n"
     ]
    }
   ],
   "source": [
    "error, total = test(model, testloader)\n",
    "print('Top-1 error of the network on test images: %0.3f %%' \\\n",
    "      % (100 * (error / total)))"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "datasetId": 2334639,
     "sourceId": 3932654,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 2632619,
     "sourceId": 4503325,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30301,
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
