diff --git a/C3W2_SRGAN_(Optional).ipynb b/C3W2_SRGAN_(Optional).ipynb
new file mode 100644
index 0000000..c29dc84
--- /dev/null
+++ b/C3W2_SRGAN_(Optional).ipynb
@@ -0,0 +1,4327 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "colab": {
+ "name": "C3W2: SRGAN (Optional).ipynb",
+ "provenance": [],
+ "collapsed_sections": [],
+ "toc_visible": true,
+ "include_colab_link": true
+ },
+ "kernelspec": {
+ "name": "python3",
+ "display_name": "Python 3"
+ },
+ "accelerator": "GPU",
+ "widgets": {
+ "application/vnd.jupyter.widget-state+json": {
+ "98e801dae0b4401785ccd06d3f42a6a0": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HBoxModel",
+ "state": {
+ "_view_name": "HBoxView",
+ "_dom_classes": [],
+ "_model_name": "HBoxModel",
+ "_view_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_view_count": null,
+ "_view_module_version": "1.5.0",
+ "box_style": "",
+ "layout": "IPY_MODEL_1dc6f457233149e6a8789446b69e6f9d",
+ "_model_module": "@jupyter-widgets/controls",
+ "children": [
+ "IPY_MODEL_b3920fc234fd402a96803bcf9382126d",
+ "IPY_MODEL_968ee790ed30446dba2b67b07376c145"
+ ]
+ }
+ },
+ "1dc6f457233149e6a8789446b69e6f9d": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "state": {
+ "_view_name": "LayoutView",
+ "grid_template_rows": null,
+ "right": null,
+ "justify_content": null,
+ "_view_module": "@jupyter-widgets/base",
+ "overflow": null,
+ "_model_module_version": "1.2.0",
+ "_view_count": null,
+ "flex_flow": null,
+ "width": null,
+ "min_width": null,
+ "border": null,
+ "align_items": null,
+ "bottom": null,
+ "_model_module": "@jupyter-widgets/base",
+ "top": null,
+ "grid_column": null,
+ "overflow_y": null,
+ "overflow_x": null,
+ "grid_auto_flow": null,
+ "grid_area": null,
+ "grid_template_columns": null,
+ "flex": null,
+ "_model_name": "LayoutModel",
+ "justify_items": null,
+ "grid_row": null,
+ "max_height": null,
+ "align_content": null,
+ "visibility": null,
+ "align_self": null,
+ "height": null,
+ "min_height": null,
+ "padding": null,
+ "grid_auto_rows": null,
+ "grid_gap": null,
+ "max_width": null,
+ "order": null,
+ "_view_module_version": "1.2.0",
+ "grid_template_areas": null,
+ "object_position": null,
+ "object_fit": null,
+ "grid_auto_columns": null,
+ "margin": null,
+ "display": null,
+ "left": null
+ }
+ },
+ "b3920fc234fd402a96803bcf9382126d": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_view_name": "ProgressView",
+ "style": "IPY_MODEL_c9485c8bb4a541079034da385d512e70",
+ "_dom_classes": [],
+ "description": "",
+ "_model_name": "FloatProgressModel",
+ "bar_style": "info",
+ "max": 1,
+ "_view_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "value": 1,
+ "_view_count": null,
+ "_view_module_version": "1.5.0",
+ "orientation": "horizontal",
+ "min": 0,
+ "description_tooltip": null,
+ "_model_module": "@jupyter-widgets/controls",
+ "layout": "IPY_MODEL_96ec84b5a00341f18bf58067f942045a"
+ }
+ },
+ "968ee790ed30446dba2b67b07376c145": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "HTMLModel",
+ "state": {
+ "_view_name": "HTMLView",
+ "style": "IPY_MODEL_0c356917abbf4afb8ef89e5e48930635",
+ "_dom_classes": [],
+ "description": "",
+ "_model_name": "HTMLModel",
+ "placeholder": "",
+ "_view_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "value": " 2640404480/? [11:50<00:00, 3518178.90it/s]",
+ "_view_count": null,
+ "_view_module_version": "1.5.0",
+ "description_tooltip": null,
+ "_model_module": "@jupyter-widgets/controls",
+ "layout": "IPY_MODEL_c0209daeab5f4a09b01a23370e09d352"
+ }
+ },
+ "c9485c8bb4a541079034da385d512e70": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_view_name": "StyleView",
+ "_model_name": "ProgressStyleModel",
+ "description_width": "initial",
+ "_view_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.5.0",
+ "_view_count": null,
+ "_view_module_version": "1.2.0",
+ "bar_color": null,
+ "_model_module": "@jupyter-widgets/controls"
+ }
+ },
+ "96ec84b5a00341f18bf58067f942045a": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "state": {
+ "_view_name": "LayoutView",
+ "grid_template_rows": null,
+ "right": null,
+ "justify_content": null,
+ "_view_module": "@jupyter-widgets/base",
+ "overflow": null,
+ "_model_module_version": "1.2.0",
+ "_view_count": null,
+ "flex_flow": null,
+ "width": null,
+ "min_width": null,
+ "border": null,
+ "align_items": null,
+ "bottom": null,
+ "_model_module": "@jupyter-widgets/base",
+ "top": null,
+ "grid_column": null,
+ "overflow_y": null,
+ "overflow_x": null,
+ "grid_auto_flow": null,
+ "grid_area": null,
+ "grid_template_columns": null,
+ "flex": null,
+ "_model_name": "LayoutModel",
+ "justify_items": null,
+ "grid_row": null,
+ "max_height": null,
+ "align_content": null,
+ "visibility": null,
+ "align_self": null,
+ "height": null,
+ "min_height": null,
+ "padding": null,
+ "grid_auto_rows": null,
+ "grid_gap": null,
+ "max_width": null,
+ "order": null,
+ "_view_module_version": "1.2.0",
+ "grid_template_areas": null,
+ "object_position": null,
+ "object_fit": null,
+ "grid_auto_columns": null,
+ "margin": null,
+ "display": null,
+ "left": null
+ }
+ },
+ "0c356917abbf4afb8ef89e5e48930635": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_view_name": "StyleView",
+ "_model_name": "DescriptionStyleModel",
+ "description_width": "",
+ "_view_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.5.0",
+ "_view_count": null,
+ "_view_module_version": "1.2.0",
+ "_model_module": "@jupyter-widgets/controls"
+ }
+ },
+ "c0209daeab5f4a09b01a23370e09d352": {
+ "model_module": "@jupyter-widgets/base",
+ "model_name": "LayoutModel",
+ "state": {
+ "_view_name": "LayoutView",
+ "grid_template_rows": null,
+ "right": null,
+ "justify_content": null,
+ "_view_module": "@jupyter-widgets/base",
+ "overflow": null,
+ "_model_module_version": "1.2.0",
+ "_view_count": null,
+ "flex_flow": null,
+ "width": null,
+ "min_width": null,
+ "border": null,
+ "align_items": null,
+ "bottom": null,
+ "_model_module": "@jupyter-widgets/base",
+ "top": null,
+ "grid_column": null,
+ "overflow_y": null,
+ "overflow_x": null,
+ "grid_auto_flow": null,
+ "grid_area": null,
+ "grid_template_columns": null,
+ "flex": null,
+ "_model_name": "LayoutModel",
+ "justify_items": null,
+ "grid_row": null,
+ "max_height": null,
+ "align_content": null,
+ "visibility": null,
+ "align_self": null,
+ "height": null,
+ "min_height": null,
+ "padding": null,
+ "grid_auto_rows": null,
+ "grid_gap": null,
+ "max_width": null,
+ "order": null,
+ "_view_module_version": "1.2.0",
+ "grid_template_areas": null,
+ "object_position": null,
+ "object_fit": null,
+ "grid_auto_columns": null,
+ "margin": null,
+ "display": null,
+ "left": null
+ }
+ }
+ }
+ }
+ },
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "view-in-github",
+ "colab_type": "text"
+ },
+ "source": [
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "1czVdIlqnImH"
+ },
+ "source": [
+ "# Super-resolution GAN (SRGAN)\n",
+ "\n",
+ "*Please note that this is an optional notebook meant to introduce more advanced concepts. If you’re up for a challenge, take a look and don’t worry if you can’t follow everything. There is no code to implement—only some cool code for you to learn and run!*\n",
+ "\n",
+ "It is recommended that you should already be familiar with:\n",
+ " - Residual blocks, from [Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385) (He et al. 2015)\n",
+ " - Perceptual loss, from [Perceptual Losses for Real-Time Style Transfer and Super-Resolution](https://arxiv.org/abs/1603.08155) (Johnson et al. 2016)\n",
+ " - VGG architecture, from [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556) (Simonyan et al. 2015)\n",
+ "\n",
+ "### Goals\n",
+ "\n",
+ "In this notebook, you will learn about Super-Resolution GAN (SRGAN), a GAN that enhances the resolution of images by 4x, proposed in [Photo-Realistic Single Image Super-Resolution Using a Generative Adversarial Network](https://arxiv.org/abs/1609.04802) (Ledig et al. 2017). You will also implement the architecture and training in full and be able to train it on the CIFAR dataset.\n",
+ "\n",
+ "### Background\n",
+ "\n",
+ "The authors first train a super-resolution residual network (SRResNet) with standard pixel-wise loss that achieves state-of-the-art metrics. They then insert this as the generator in the SRGAN framework, which is trained with a combination of pixel-wise, perceptual, and adversarial losses."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "_SeMjdGIICPB"
+ },
+ "source": [
+ "## SRGAN Submodules\n",
+ "\n",
+ "Before jumping into SRGAN, let's first take a look at some components that will be useful later."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "s9zFUlwLIOcC"
+ },
+ "source": [
+ "### Parametric ReLU (PReLU)\n",
+ "\n",
+ "As you already know, ReLU is one of the simplest activation functions that can be described as\n",
+ "\n",
+ "\\begin{align*}\n",
+ " x_{\\text{ReLU}} := \\max(0, x),\n",
+ "\\end{align*}\n",
+ "\n",
+ "where negative values of $x$ become thresholded at $0$. However, this stops gradient through these negative values, which can hinder training. The authors of [Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification](https://arxiv.org/abs/1502.01852) addressed this by introducing a more general ReLU by scaling negative values by some constant $a > 0$:\n",
+ "\n",
+ "\\begin{align*}\n",
+ " x_{\\text{PReLU}} := \\max(0, x) + a * \\min(0, x).\n",
+ "\\end{align*}\n",
+ "\n",
+ "Conveniently, this is implemented in Pytorch as [torch.nn.PReLU](https://pytorch.org/docs/stable/generated/torch.nn.PReLU.html)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "fB1Vq8ps7Bfd"
+ },
+ "source": [
+ "### Residual Blocks\n",
+ "\n",
+ "The residual block, which is relevant in many state-of-the-art computer vision models, is used in all parts of SRGAN and is similar to the ones used in Pix2PixHD (see optional notebook). If you're not familiar with residual blocks, please take a look [here](https://paperswithcode.com/method/residual-block). Now, you'll start by first implementing a basic residual block."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "GHD_wif07f4b"
+ },
+ "source": [
+ "import torch\n",
+ "import torch.nn as nn\n",
+ "import torch.nn.functional as F\n",
+ "\n",
+ "class ResidualBlock(nn.Module):\n",
+ " '''\n",
+ " ResidualBlock Class\n",
+ " Values\n",
+ " channels: the number of channels throughout the residual block, a scalar\n",
+ " '''\n",
+ "\n",
+ " def __init__(self, channels):\n",
+ " super().__init__()\n",
+ "\n",
+ " self.layers = nn.Sequential(\n",
+ " nn.Conv2d(channels, channels, kernel_size=3, padding=1),\n",
+ " nn.BatchNorm2d(channels),\n",
+ " nn.PReLU(),\n",
+ "\n",
+ " nn.Conv2d(channels, channels, kernel_size=3, padding=1),\n",
+ " nn.BatchNorm2d(channels),\n",
+ " )\n",
+ "\n",
+ " def forward(self, x):\n",
+ " return x + self.layers(x)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "cCBCYvalE04U"
+ },
+ "source": [
+ "### PixelShuffle\n",
+ "\n",
+ "Proposed in [Real-Time Single Image and Video Super-Resolution Using an Efficient Sub-Pixel Convolutional Neural Network](https://arxiv.org/abs/1609.05158) (Shi et al. 2016), PixelShuffle, also called sub-pixel convolution, is another way to upsample an image.\n",
+ "\n",
+ "PixelShuffle simply reshapes a $r^2C\\ x\\ H\\ x\\ W$ tensor into a $C\\ x\\ rH\\ x\\ rW$ tensor, essentially trading channel information for spatial information. Instead of convolving with stride $1/r$ as in deconvolution, the authors think about the weights in the kernel as being spaced $1/r$ pixels apart. When sliding this kernel over an input, the weights that fall between pixels aren't activated and don't need need to be calculated. The total number of activation patterns is thus increased by a factor of $r^2$. This operation is illustrated in the figure below.\n",
+ "\n",
+ "Don't worry if this is confusing! The algorithm is conveniently implemented as `torch.nn.PixelShuffle` in PyTorch, so as long as you have a general idea of how this works, you're set.\n",
+ "\n",
+ "> ![Efficient Sub-pixel CNN](https://drive.google.com/uc?export=view&id=136LMcywv1r5W1f9-L-55bJ4AKAHlH8Zr)\n",
+ "*Efficient sub-pixel CNN, taken from Figure 1 of [Real-Time Single Image and Video Super-Resolution Using an Efficient Sub-Pixel Convolutional Neural Network](https://arxiv.org/abs/1609.05158) (Shi et al. 2016). The PixelShuffle operation (also known as sub-pixel convolution) is shown as the last step on the right.*"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "bDSSM7SJJgon"
+ },
+ "source": [
+ "## SRGAN Parts\n",
+ "\n",
+ "Now that you've learned about the various SRGAN submodules, you can now use them to build the generator and discriminator!"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "vZ41VxyyIaes"
+ },
+ "source": [
+ "### Generator (SRResNet)\n",
+ "\n",
+ "The super-resolution residual network (SRResNet) and the generator are the same thing. The generator network architecture is actually quite simple - just a bunch of convolutional layers, residual blocks, and pixel shuffling layers!\n",
+ "\n",
+ "> ![SRGAN Generator](https://drive.google.com/uc?export=view&id=1wY5YmoYTBzuhWLlTAYCI92xWgkf_uINE)\n",
+ "*SRGAN Generator, taken from Figure 4 of [Photo-Realistic Single Image Super-Resolution Using a Generative Adversarial Network](https://arxiv.org/abs/1609.04802) (Ledig et al. 2017).*"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "EXqRw0f7KO81"
+ },
+ "source": [
+ "class Generator(nn.Module):\n",
+ " '''\n",
+ " Generator Class\n",
+ " Values:\n",
+ " base_channels: number of channels throughout the generator, a scalar\n",
+ " n_ps_blocks: number of PixelShuffle blocks, a scalar\n",
+ " n_res_blocks: number of residual blocks, a scalar\n",
+ " '''\n",
+ "\n",
+ " def __init__(self, base_channels=64, n_ps_blocks=2, n_res_blocks=16):\n",
+ " super().__init__()\n",
+ " # Input layer\n",
+ " self.in_layer = nn.Sequential(\n",
+ " nn.Conv2d(3, base_channels, kernel_size=9, padding=4),\n",
+ " nn.PReLU(),\n",
+ " )\n",
+ "\n",
+ " # Residual blocks\n",
+ " res_blocks = []\n",
+ " for _ in range(n_res_blocks):\n",
+ " res_blocks += [ResidualBlock(base_channels)]\n",
+ "\n",
+ " res_blocks += [\n",
+ " nn.Conv2d(base_channels, base_channels, kernel_size=3, padding=1),\n",
+ " nn.BatchNorm2d(base_channels),\n",
+ " ]\n",
+ " self.res_blocks = nn.Sequential(*res_blocks)\n",
+ "\n",
+ " # PixelShuffle blocks\n",
+ " ps_blocks = []\n",
+ " for _ in range(n_ps_blocks):\n",
+ " ps_blocks += [\n",
+ " nn.Conv2d(base_channels, 4 * base_channels, kernel_size=3, padding=1),\n",
+ " nn.PixelShuffle(2),\n",
+ " nn.PReLU(),\n",
+ " ]\n",
+ " self.ps_blocks = nn.Sequential(*ps_blocks)\n",
+ "\n",
+ " # Output layer\n",
+ " self.out_layer = nn.Sequential(\n",
+ " nn.Conv2d(base_channels, 3, kernel_size=9, padding=4),\n",
+ " nn.Tanh(),\n",
+ " )\n",
+ "\n",
+ " def forward(self, x):\n",
+ " x_res = self.in_layer(x)\n",
+ " x = x_res + self.res_blocks(x_res)\n",
+ " x = self.ps_blocks(x)\n",
+ " x = self.out_layer(x)\n",
+ " return x"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "mbpiSiWpNjtR"
+ },
+ "source": [
+ "### Discriminator\n",
+ "\n",
+ "The discriminator architecture is also relatively straightforward, just one big sequential model - see the diagram below for reference!\n",
+ "\n",
+ "![SRGAN Generator](https://drive.google.com/uc?export=view&id=1fcfTrXBcODoZa2JSEO8OiMUEo5WHdkef)\n",
+ "*SRGAN Discriminator, taken from Figure 4 of [Photo-Realistic Single Image Super-Resolution Using a Generative Adversarial Network](https://arxiv.org/abs/1609.04802) (Ledig et al. 2017).*"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "cpyEsLi-OQj4"
+ },
+ "source": [
+ "class Discriminator(nn.Module):\n",
+ " '''\n",
+ " Discriminator Class\n",
+ " Values:\n",
+ " base_channels: number of channels in first convolutional layer, a scalar\n",
+ " n_blocks: number of convolutional blocks, a scalar\n",
+ " '''\n",
+ "\n",
+ " def __init__(self, base_channels=64, n_blocks=3):\n",
+ " super().__init__()\n",
+ " self.blocks = [\n",
+ " nn.Conv2d(3, base_channels, kernel_size=3, padding=1),\n",
+ " nn.LeakyReLU(0.2, inplace=True),\n",
+ "\n",
+ " nn.Conv2d(base_channels, base_channels, kernel_size=3, padding=1, stride=2),\n",
+ " nn.BatchNorm2d(base_channels),\n",
+ " nn.LeakyReLU(0.2, inplace=True),\n",
+ " ]\n",
+ "\n",
+ " cur_channels = base_channels\n",
+ " for i in range(n_blocks):\n",
+ " self.blocks += [\n",
+ " nn.Conv2d(cur_channels, 2 * cur_channels, kernel_size=3, padding=1),\n",
+ " nn.BatchNorm2d(2 * cur_channels),\n",
+ " nn.LeakyReLU(0.2, inplace=True),\n",
+ "\n",
+ " nn.Conv2d(2 * cur_channels, 2 * cur_channels, kernel_size=3, padding=1, stride=2),\n",
+ " nn.BatchNorm2d(2 * cur_channels),\n",
+ " nn.LeakyReLU(0.2, inplace=True),\n",
+ " ]\n",
+ " cur_channels *= 2\n",
+ "\n",
+ " self.blocks += [\n",
+ " # You can replicate nn.Linear with pointwise nn.Conv2d\n",
+ " nn.AdaptiveAvgPool2d(1),\n",
+ " nn.Conv2d(cur_channels, 2 * cur_channels, kernel_size=1, padding=0),\n",
+ " nn.LeakyReLU(0.2, inplace=True),\n",
+ " nn.Conv2d(2 * cur_channels, 1, kernel_size=1, padding=0),\n",
+ "\n",
+ " # Apply sigmoid if necessary in loss function for stability\n",
+ " nn.Flatten(),\n",
+ " ]\n",
+ "\n",
+ " self.layers = nn.Sequential(*self.blocks)\n",
+ "\n",
+ " def forward(self, x):\n",
+ " return self.layers(x)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "qzkNr4vgTGvb"
+ },
+ "source": [
+ "## Loss Functions\n",
+ "\n",
+ "The authors formulate the perceptual loss as a weighted sum of content loss (based on the VGG19 network) and adversarial loss.\n",
+ "\n",
+ "\\begin{align*}\n",
+ " \\mathcal{L} &= \\mathcal{L}_{VGG} + 10^{-3}\\mathcal{L}_{ADV}\n",
+ "\\end{align*}\n",
+ "\n",
+ "**Content Loss**\n",
+ "\n",
+ "Previous approaches have used MSE loss for content loss, but this objective function tends to produce blurry images. To address this, they add an extra MSE loss term on VGG19 feature maps. So for feature map $\\phi_{5,4}$ (the feature map after the 4th convolution before the 5th max-pooling layer) from the VGG19 network,\n",
+ "\n",
+ "\\begin{align*}\n",
+ " \\mathcal{L}_{VGG} &= \\left|\\left|\\phi_{5,4}(I^{\\text{HR}}) - \\phi_{5,4}(G(I^{\\text{LR}}))\\right|\\right|_2^2\n",
+ "\\end{align*}\n",
+ "\n",
+ "where $I^{\\text{HR}}$ is the original high-resolution image and $I^{\\text{LR}}$ is the corresponding low-resolution image.\n",
+ "\n",
+ "**Adversarial Loss**\n",
+ "\n",
+ "You should already be familiar with adversarial loss, which is formulated as\n",
+ "\n",
+ "\\begin{align*}\n",
+ " \\mathcal{L}_{ADV} &= \\sum_{n=1}^N -\\log D(G(I^{\\text{LR}}))\n",
+ "\\end{align*}\n",
+ "\n",
+ "Note that $-\\log D(G(\\cdot))$ is used instead of $\\log [1 - D(G(\\cdot))]$ for better gradient behavior."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "Mv2KIqHgJ3gh"
+ },
+ "source": [
+ "from torchvision.models import vgg19\n",
+ "\n",
+ "class Loss(nn.Module):\n",
+ " '''\n",
+ " Loss Class\n",
+ " Implements composite content+adversarial loss for SRGAN\n",
+ " Values:\n",
+ " device: 'cuda' or 'cpu' hardware to put VGG network on, a string\n",
+ " '''\n",
+ "\n",
+ " def __init__(self, device='cuda'):\n",
+ " super().__init__()\n",
+ "\n",
+ " vgg = vgg19(pretrained=True).to(device)\n",
+ " self.vgg = nn.Sequential(*list(vgg.features)[:-1]).eval()\n",
+ " for p in self.vgg.parameters():\n",
+ " p.requires_grad = False\n",
+ "\n",
+ " @staticmethod\n",
+ " def img_loss(x_real, x_fake):\n",
+ " return F.mse_loss(x_real, x_fake)\n",
+ "\n",
+ " def adv_loss(self, x, is_real):\n",
+ " target = torch.zeros_like(x) if is_real else torch.ones_like(x)\n",
+ " return F.binary_cross_entropy_with_logits(x, target)\n",
+ "\n",
+ " def vgg_loss(self, x_real, x_fake):\n",
+ " return F.mse_loss(self.vgg(x_real), self.vgg(x_fake))\n",
+ "\n",
+ " def forward(self, generator, discriminator, hr_real, lr_real):\n",
+ " ''' Performs forward pass and returns total losses for G and D '''\n",
+ " hr_fake = generator(lr_real)\n",
+ " fake_preds_for_g = discriminator(hr_fake)\n",
+ " fake_preds_for_d = discriminator(hr_fake.detach())\n",
+ " real_preds_for_d = discriminator(hr_real.detach())\n",
+ "\n",
+ " g_loss = (\n",
+ " 0.001 * self.adv_loss(fake_preds_for_g, False) + \\\n",
+ " 0.006 * self.vgg_loss(hr_real, hr_fake) + \\\n",
+ " self.img_loss(hr_real, hr_fake)\n",
+ " )\n",
+ " d_loss = 0.5 * (\n",
+ " self.adv_loss(real_preds_for_d, True) + \\\n",
+ " self.adv_loss(fake_preds_for_d, False)\n",
+ " )\n",
+ "\n",
+ " return g_loss, d_loss, hr_fake"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "yAYCRF1cS8DM"
+ },
+ "source": [
+ "## Training SRGAN\n",
+ "\n",
+ "Now it's time to train your SRGAN! Let's first begin by defining our dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "DaiSLB8UTFWV"
+ },
+ "source": [
+ "from PIL import Image\n",
+ "import numpy as np\n",
+ "import torchvision\n",
+ "import torchvision.transforms as transforms\n",
+ "\n",
+ "# We are using STL (for speed and also since ImageNet is no longer publicly available)\n",
+ "USING_STL = True\n",
+ "if USING_STL:\n",
+ " DatasetSubclass = torchvision.datasets.STL10\n",
+ "else:\n",
+ " DatasetSubclass = torchvision.datasets.ImageNet\n",
+ "\n",
+ "class Dataset(DatasetSubclass):\n",
+ " '''\n",
+ " Dataset Class\n",
+ " Implements a general dataset class for STL10 and ImageNet\n",
+ " Values:\n",
+ " hr_size: spatial size of high-resolution image, a list/tuple\n",
+ " lr_size: spatial size of low-resolution image, a list/tuple\n",
+ " *args/**kwargs: all other arguments for subclassed torchvision dataset\n",
+ " '''\n",
+ "\n",
+ " def __init__(self, *args, **kwargs):\n",
+ " hr_size = kwargs.pop('hr_size', [96, 96])\n",
+ " lr_size = kwargs.pop('lr_size', [24, 24])\n",
+ " super().__init__(*args, **kwargs)\n",
+ "\n",
+ " if hr_size is not None and lr_size is not None:\n",
+ " assert hr_size[0] == 4 * lr_size[0]\n",
+ " assert hr_size[1] == 4 * lr_size[1]\n",
+ "\n",
+ " # High-res images are cropped and scaled to [-1, 1]\n",
+ " self.hr_transforms = transforms.Compose([\n",
+ " transforms.RandomCrop(hr_size),\n",
+ " transforms.RandomHorizontalFlip(),\n",
+ " transforms.Lambda(lambda img: np.array(img)),\n",
+ " transforms.ToTensor(),\n",
+ " transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n",
+ " ])\n",
+ "\n",
+ " # Low-res images are downsampled with bicubic kernel and scaled to [0, 1]\n",
+ " self.lr_transforms = transforms.Compose([\n",
+ " transforms.Normalize((-1.0, -1.0, -1.0), (2.0, 2.0, 2.0)),\n",
+ " transforms.ToPILImage(),\n",
+ " transforms.Resize(lr_size, interpolation=Image.BICUBIC),\n",
+ " transforms.ToTensor(),\n",
+ " ])\n",
+ "\n",
+ " self.to_pil = transforms.ToPILImage()\n",
+ " self.to_tensor = transforms.ToTensor()\n",
+ "\n",
+ " def __getitem__(self, idx):\n",
+ " # Uncomment the following lines if you're using ImageNet\n",
+ " # path, label = self.imgs[idx]\n",
+ " # image = Image.open(path).convert('RGB')\n",
+ "\n",
+ " # Uncomment the following if you're using STL\n",
+ " image = torch.from_numpy(self.data[idx])\n",
+ " image = self.to_pil(image)\n",
+ "\n",
+ " hr = self.hr_transforms(image)\n",
+ " lr = self.lr_transforms(hr)\n",
+ " return hr, lr\n",
+ "\n",
+ " @staticmethod\n",
+ " def collate_fn(batch):\n",
+ " hrs, lrs = [], []\n",
+ "\n",
+ " for hr, lr in batch:\n",
+ " hrs.append(hr)\n",
+ " lrs.append(lr)\n",
+ "\n",
+ " return torch.stack(hrs, dim=0), torch.stack(lrs, dim=0)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "cATrAvDhaq6c"
+ },
+ "source": [
+ "Recall that the generator (SRResNet) is first trained alone with MSE loss and is combined with the discriminator and trained as SRGAN after. Check out the training loops below:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "FTEi1FAgbCQG"
+ },
+ "source": [
+ "from tqdm import tqdm\n",
+ "from torchvision.utils import make_grid\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "# Parse torch version for autocast\n",
+ "# ######################################################\n",
+ "version = torch.__version__\n",
+ "version = tuple(int(n) for n in version.split('.')[:-1])\n",
+ "has_autocast = version >= (1, 6)\n",
+ "# ######################################################\n",
+ "\n",
+ "def show_tensor_images(image_tensor):\n",
+ " '''\n",
+ " Function for visualizing images: Given a tensor of images, number of images, and\n",
+ " size per image, plots and prints the images in an uniform grid.\n",
+ " '''\n",
+ " image_tensor = (image_tensor + 1) / 2\n",
+ " image_unflat = image_tensor.detach().cpu()\n",
+ " image_grid = make_grid(image_unflat[:4], nrow=4)\n",
+ " plt.axis('off')\n",
+ " plt.imshow(image_grid.permute(1, 2, 0).squeeze())\n",
+ " plt.show()\n",
+ "\n",
+ "def train_srresnet(srresnet, dataloader, device, lr=1e-4, total_steps=1e6, display_step=500):\n",
+ " srresnet = srresnet.to(device).train()\n",
+ " optimizer = torch.optim.Adam(srresnet.parameters(), lr=lr)\n",
+ "\n",
+ " cur_step = 0\n",
+ " mean_loss = 0.0\n",
+ " while cur_step < total_steps:\n",
+ " for hr_real, lr_real in tqdm(dataloader, position=0):\n",
+ " hr_real = hr_real.to(device)\n",
+ " lr_real = lr_real.to(device)\n",
+ "\n",
+ " # Enable autocast to FP16 tensors (new feature since torch==1.6.0)\n",
+ " # If you're running older versions of torch, comment this out\n",
+ " # and use NVIDIA apex for mixed/half precision training\n",
+ " if has_autocast:\n",
+ " with torch.cuda.amp.autocast(enabled=(device=='cuda')):\n",
+ " hr_fake = srresnet(lr_real)\n",
+ " loss = Loss.img_loss(hr_real, hr_fake)\n",
+ " else:\n",
+ " hr_fake = srresnet(lr_real)\n",
+ " loss = Loss.img_loss(hr_real, hr_fake)\n",
+ " \n",
+ " optimizer.zero_grad()\n",
+ " loss.backward()\n",
+ " optimizer.step()\n",
+ "\n",
+ " mean_loss += loss.item() / display_step\n",
+ "\n",
+ " if cur_step % display_step == 0 and cur_step > 0:\n",
+ " print('Step {}: SRResNet loss: {:.5f}'.format(cur_step, mean_loss))\n",
+ " show_tensor_images(lr_real * 2 - 1)\n",
+ " show_tensor_images(hr_fake.to(hr_real.dtype))\n",
+ " show_tensor_images(hr_real)\n",
+ " mean_loss = 0.0\n",
+ "\n",
+ " cur_step += 1\n",
+ " if cur_step == total_steps:\n",
+ " break\n",
+ "\n",
+ "def train_srgan(generator, discriminator, dataloader, device, lr=1e-4, total_steps=2e5, display_step=500):\n",
+ " generator = generator.to(device).train()\n",
+ " discriminator = discriminator.to(device).train()\n",
+ " loss_fn = Loss(device=device)\n",
+ "\n",
+ " g_optimizer = torch.optim.Adam(generator.parameters(), lr=lr)\n",
+ " d_optimizer = torch.optim.Adam(discriminator.parameters(), lr=lr)\n",
+ " g_scheduler = torch.optim.lr_scheduler.LambdaLR(g_optimizer, lambda _: 0.1)\n",
+ " d_scheduler = torch.optim.lr_scheduler.LambdaLR(d_optimizer, lambda _: 0.1)\n",
+ "\n",
+ " lr_step = total_steps // 2\n",
+ " cur_step = 0\n",
+ "\n",
+ " mean_g_loss = 0.0\n",
+ " mean_d_loss = 0.0\n",
+ "\n",
+ " while cur_step < total_steps:\n",
+ " for hr_real, lr_real in tqdm(dataloader, position=0):\n",
+ " hr_real = hr_real.to(device)\n",
+ " lr_real = lr_real.to(device)\n",
+ "\n",
+ " # Enable autocast to FP16 tensors (new feature since torch==1.6.0)\n",
+ " # If you're running older versions of torch, comment this out\n",
+ " # and use NVIDIA apex for mixed/half precision training\n",
+ " if has_autocast:\n",
+ " with torch.cuda.amp.autocast(enabled=(device=='cuda')):\n",
+ " g_loss, d_loss, hr_fake = loss_fn(\n",
+ " generator, discriminator, hr_real, lr_real,\n",
+ " )\n",
+ " else:\n",
+ " g_loss, d_loss, hr_fake = loss_fn(\n",
+ " generator, discriminator, hr_real, lr_real,\n",
+ " )\n",
+ "\n",
+ " g_optimizer.zero_grad()\n",
+ " g_loss.backward()\n",
+ " g_optimizer.step()\n",
+ "\n",
+ " d_optimizer.zero_grad()\n",
+ " d_loss.backward()\n",
+ " d_optimizer.step()\n",
+ "\n",
+ " mean_g_loss += g_loss.item() / display_step\n",
+ " mean_d_loss += d_loss.item() / display_step\n",
+ "\n",
+ " if cur_step == lr_step:\n",
+ " g_scheduler.step()\n",
+ " d_scheduler.step()\n",
+ " print('Decayed learning rate by 10x.')\n",
+ "\n",
+ " if cur_step % display_step == 0 and cur_step > 0:\n",
+ " print('Step {}: Generator loss: {:.5f}, Discriminator loss: {:.5f}'.format(cur_step, mean_g_loss, mean_d_loss))\n",
+ " show_tensor_images(lr_real * 2 - 1)\n",
+ " show_tensor_images(hr_fake.to(hr_real.dtype))\n",
+ " show_tensor_images(hr_real)\n",
+ " mean_g_loss = 0.0\n",
+ " mean_d_loss = 0.0\n",
+ "\n",
+ " cur_step += 1\n",
+ " if cur_step == total_steps:\n",
+ " break"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nyypDt_UhBjz"
+ },
+ "source": [
+ "Now initialize everything and run training!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "ScH0Iok8fAMS",
+ "outputId": "388ba277-76ce-427b-b7b1-57ab74e9b1dd",
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 84,
+ "referenced_widgets": [
+ "98e801dae0b4401785ccd06d3f42a6a0",
+ "1dc6f457233149e6a8789446b69e6f9d",
+ "b3920fc234fd402a96803bcf9382126d",
+ "968ee790ed30446dba2b67b07376c145",
+ "c9485c8bb4a541079034da385d512e70",
+ "96ec84b5a00341f18bf58067f942045a",
+ "0c356917abbf4afb8ef89e5e48930635",
+ "c0209daeab5f4a09b01a23370e09d352"
+ ]
+ }
+ },
+ "source": [
+ "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
+ "generator = Generator(n_res_blocks=16, n_ps_blocks=2)\n",
+ "\n",
+ "# Uncomment the following lines if you're using ImageNet\n",
+ "# dataloader = torch.utils.data.DataLoader(\n",
+ "# Dataset('data', 'train', download=True, hr_size=[384, 384], lr_size=[96, 96]),\n",
+ "# batch_size=16, pin_memory=True, shuffle=True,\n",
+ "# )\n",
+ "# train_srresnet(generator, dataloader, device, lr=1e-4, total_steps=1e6, display_step=500)\n",
+ "# torch.save(generator, 'srresnet.pt')\n",
+ "\n",
+ "# Uncomment the following lines if you're using STL\n",
+ "dataloader = torch.utils.data.DataLoader(\n",
+ " Dataset('data', 'train', download=True, hr_size=[96, 96], lr_size=[24, 24]),\n",
+ " batch_size=16, pin_memory=True, shuffle=True,\n",
+ ")\n",
+ "train_srresnet(generator, dataloader, device, lr=1e-4, total_steps=1e5, display_step=1000)\n",
+ "torch.save(generator, 'srresnet.pt')"
+ ],
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "text": [
+ "Downloading http://ai.stanford.edu/~acoates/stl10/stl10_binary.tar.gz to data/stl10_binary.tar.gz\n"
+ ],
+ "name": "stdout"
+ },
+ {
+ "output_type": "display_data",
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "98e801dae0b4401785ccd06d3f42a6a0",
+ "version_minor": 0,
+ "version_major": 2
+ },
+ "text/plain": [
+ "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ }
+ },
+ {
+ "output_type": "stream",
+ "text": [
+ "Extracting data/stl10_binary.tar.gz to data\n"
+ ],
+ "name": "stdout"
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "AvqBuRwjZrBq",
+ "outputId": "96ff8cef-0371-4bb3-d4ff-b1c1aee180b6",
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 1000
+ }
+ },
+ "source": [
+ "generator = torch.load('srresnet.pt')\n",
+ "discriminator = Discriminator(n_blocks=1, base_channels=8)\n",
+ "\n",
+ "# Uncomment the following lines if you're using ImageNet\n",
+ "# train_srgan(generator, discriminator, dataloader, device, lr=1e-4, total_steps=2e5, display_step=500)\n",
+ "# torch.save(generator, 'srgenerator.pt')\n",
+ "# torch.save(discriminator, 'srdiscriminator.pt')\n",
+ "\n",
+ "# Uncomment the following lines if you're using STL\n",
+ "train_srgan(generator, discriminator, dataloader, device, lr=1e-4, total_steps=2e5, display_step=1000)\n",
+ "torch.save(generator, 'srgenerator.pt')\n",
+ "torch.save(discriminator, 'srdiscriminator.pt')"
+ ],
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "text": [
+ "100%|██████████| 313/313 [00:38<00:00, 8.16it/s]\n",
+ "100%|██████████| 313/313 [00:38<00:00, 8.09it/s]\n",
+ "100%|██████████| 313/313 [00:38<00:00, 8.13it/s]\n",
+ " 19%|█▉ | 61/313 [00:07<00:30, 8.33it/s]"
+ ],
+ "name": "stderr"
+ },
+ {
+ "output_type": "stream",
+ "text": [
+ "Step 1000: Generator loss: 0.01806, Discriminator loss: 0.69390\n"
+ ],
+ "name": "stdout"
+ },
+ {
+ "output_type": "display_data",
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAABmCAYAAABoQkJtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO19aYwk53neW1VdXX13z/Tcszu7Ozt7kLu8RMqkKEp0TCsJHUdyAsQBjNwCZDiwEeRPEBixf9hxEiAB8iNRgsRwZBuCAyNSlEMCHUqyLEWiTJvSknuQ3IN7zOzcZ0/f3XXkR6B6nq88vTscGm07eZ9f73RX1/lVTT3P97zva0VRJAqFQqEYDuw/6R1QKBSK/5+gD12FQqEYIvShq1AoFEOEPnQVCoViiNCHrkKhUAwRqQd9aVmWWhsUCoXifSKKImvQd/qmq1AoFEOEPnQVCoViiNCHrkKhUAwR+tBVKBSKIUIfugqFQjFEPNC9MAinz8/EcafdN77b3W7gD5rAcxw8323HieMwDGl50yyRcrGc3w/wmwDLpdJYJlvMxHGflhcRcV0catp1sa6+j/1KYR+jCPvV69M+ikguN4Llgi62n8N6R/KFOH7t9y/JQZifHTf+Xt6qYV/oHJ0YL8XxT/3VvxbHo2O4Dr1O21jXK69+MY7bre04LmSwXtvKxfHKXiuOpybxuYjIxCTOq3ToHAmury84j4XyRByXSseNdd258T38YXlx2Orh+jz/Qy/GccbDtv/Rr/yyDMLnfvXfxrHn4Rg3aztx/M3vXYnj06fm4zgdmWNlfWUjjk+dwDVq93CO9ur4zUsffz6Or1y7aqyrOopz8ei5hTi2Qvx+p7YXx7Mzx+L46pXLxrqKuXwcX3zyuTgulMo4lnQ2jpfX1rCNVcQiIp94+WVR/MlA33QVCoViiNCHrkKhUAwR+tBVKBSKIeJImu7cFLTE1dVd47vCNPQl24X2urkDbc12sNkggLbVaUAfFRFpt3pxbJF+aFPMOXORD+01CkwdViz83el24tjlffGxsoC1ZjG1Zr/XjGNerFfD/luDE1KwVstcxrLpuOgri/aRde+IdMFIzOMt56Ht2SH+t2Y9xH7AOju2MTI5Z6xrfKYSx2EfOxaFiEPSdLM5aNCV8pSxrn53gf7C9vdquNZpN41FDpkT+d7dpTg+fxY6ckjj64kz2HYWcrJsbdeNdbU70G5v3b4fx889eyGOj4XQjYMernu/Z+rDi0srcfzMhz6E5brYht+FHk9ytIyUcR5FRLwsNF3fx1xKbQ+acHXMpV/g5JVGy3IYfPylx+LYdjCmfMFxWXTdPNd8hNj0GudjSEjGw37ZwuMRx7S2heMQEekFOMa0h7GWoZNk0wYdumksm06kiIR0f/i0Xn6upGgeJaJzl6w5HtJ8z6tfMnX3w0DfdBUKhWKI0IeuQqFQDBFHkxemq3H8mb/9c8Z3lRJo7X//ypfj+Ctf+7047odkbyJ6kMkQ5xPTZtZsg8K5NtueQA96ZBNzXZNeZMl6FIagC50+aG2hBKuUlwYdKhRxTCIiThrb3NqAzSvlYBuu+/BTO1Iy6WMqxbQa+zhaBL13HJwjJ0WUjbmciDS7oEDdPvaF63CkUkynANcriBh/j2I5kozCgOUFbC/l4TymsuYxBikcS9oleaO1f+C+iP1wmUZE5FvffSOOJyewzQyNg/t7oPqlGcgejmVKM06EMXFydjqOd3ZxrXf2sMzY2GQc39uAjCYi4nkYO4aURpLE4iZ+c/48xsC1q+8a6zp9+kQcT08jLlWKcczX1KW41zZlj0F47qXTccwSlx+RtZJe1Twes4kfMXXni2rTfR2EWO98H1ZMERGH6H4um6bPMe5TAyQ5cUxJwKFzwZfbp/smINmAbalhwlLo+yovKBQKxZ8Z6ENXoVAohogjyQsLTzwRxz/2l37M+M62MDP4kRc+Fsd/5VOgf//tlS/F8Z1lUCjbNakkGxACet3fXgXN21gFNTv9KGatWSoQEclkQPP8DvbRT2EjoxP4zcQ44nyeZ4RFIqYkRDUadbgixsYxK/vW996Wg7AwZ2ZrZchZ4BCHCwOmY6A6fh8UlWezRUQKOcxWj1TpvIwgw6pHFDfYwnl0XNBlEZEgoplvsos4aQyflEWuiBDU2wnNaxp1sZ+pLGSMQhEUOQwPR4UZU2PksPBxHULita6N/d0kSh/ZlHEnItVJSA/bdUhhBXK3bKxgpv07rd+P475vvsfkqzh3y+SS2N3G9pdoPL/yjW9j3zMYQyIiV24uYx8nIC+QWia5HOSn7R04i4LE+BgEi84Xu4Q8ovQW8/iES8cSljcoe5E4fYquQ0TLF7KJd0AaawFRfHbKhPR7obEW9cz9agW4jhGdMD4WVrJ8koJsy9wvv//+xydD33QVCoViiNCHrkKhUAwRR5IX1lZW4/j2vRXjuxTNGi7feSeOJ0dBJf/GJz8Vx33vk3H8+ru/a6zr1uItrJekB78PY3m3ifVe/DBoseMmqQrFNqhSu8FUgai7DzrSaPSEwbPC7DLwsjid3Y75mwPhmLTWyuD3TIuj7mDT9w/AJnERkblzMLk7JSSzTJ+Dwf/tP/huHGea2LafMmltK8T5YtN4itwLmTwcAxHJFpubZvJMuQAZoUfDz+5DEuBZb2MG/AFI0Yx4hoog8az7dh3jZqKK/QgScsa9e6DxaZpBv7GFwkF7u5AKggjSVaZszsAX6tivpREUnbGLmI3P0/5ev343jsfG4BISEbm3ikI8O6++GsdPUSGdZ+SZOF5fwXFs7ZnXYRBG8rj2Nl2fFDtNmNGHpvPDSJxIQergRBqWCvok1bHzQsR0DfQpoaHPBajo3HGRq9BPuBdcKoYlWBcnFTmUIGQZj0bz3kqlP1gXM33TVSgUiiFCH7oKhUIxRBxJXrhy7Xocry8vGd9Vq6BXq0t3EN/YjGOu+bnwyONx/ONPmTU+LxdvxnGjDXq0+Q5mi7es9+J4Yw2zwMl86QLRpmIBVLhM+zIxhlnrbAG0ZWUL+fciIrU9MsBbTE9Aodqdh1OQpFRQqYCOsQF7r0M1Fmi1AVGzJM2bPQ6Te11wvLSLUiKZpkwGe+/EGWNdKaJjna17iNvYfsul85gl6t5M1DVokoxQwPHa5GrgYw8P2ZC6WAB9dGgauke1EDyij0xFVy8vGuvqtLH9XBv1oVOUKDFehXvAXoTsYPXM6xCQJPHuG6/Fcc2CfOWW4bzI2jinzVXIayIiWQ/31mYbv79FtZDvfAsuIbeFcTozdrjaC00bcl2fjoWlJJaYkjP7LiXs1Inih7ScZUFaCciilPHNRAs74vrW+Jyvne+zbIExlHUSjiNyMxgpWBYlTZDK1O1CHsx7ZnKU/QFfVfVNV6FQKIYIfegqFArFEHEkeaG9Acpo9c1yjGsrkBuWSXpobIDCjRZB9R2irtNEiUVExqns29kJULuP/fxfiOPX//D7cVxrQl6YPgb6JyIydwIzvIUCqFaKqNLYCLU9oToQ+wmKvLyCY/neW3AAvHvjrTi+u3JHHoZu3zSs93xs3yLa4wcw4nd7RHtC0J6+b9La62+hLc545WQcN1fhNmncgGxSptoU4xNmib0aleVcv3k7jgsFbD9dx/an2qC78ugpY11LfayrHGCbvRwlg7CEcsjajucuIGGH8xO6lCix24BUkMmjnkQzcRfYxCZrRC1bTRzXyhKSeu4vYTw89yzKN4qIlKYhHbx2CXn6KTLlz5yC62asgKScXmTW0+hbOJYeyUnpEaojMQVpyK9t4Th2IcM9CMEmrklE8oKbwee2w0kTCUmAk3poHyOSfHa2yUlBFztbNiUBj8akUYeBHC3tFs5JAxVXZcm8ZWV8nO5tGzpCr4Mx0Wmi/kdtHytI1lEJP5h5Qd90FQqFYpjQh65CoVAMEfrQVSgUiiHiSJru9AlYV1ZuvmN8d/ke9K1uB/pdziO9krKX0lSYxcuZdVwnqjO0HL6rkLXr5b+IAi5cJ9dN1HF1qXhIjbJzlhbZAgUdJ0vbGxtHG20RkUcfeTqOq2Vs/8WnPhLHn/v1X6dfvCYHoZfQdPs+t4DBdxeffDKOK3lofqsr0GfTnlngp9uB1r7wHlrAP0X638gC2vJwsY9+xxTEArLPdZ5E6/JaCxrnvZuwN32b2secN0sky/ZtaKENsuvVSeMsT43FsTeLluQPglfAcaXL0Gvn5rADC4+/FMc7m8jWymShU4uIOBZui50aNL/9TWRinp7F9vrPoh16slhPkVvuhND8fcq+atSxjTGqG1uZMFsd3VpHRtp6HQJm59UvxvHkiXNxXKpiHiSVOdyt3tzYO/DztkDvDGjcJrtiRVSTlu1gYmSh4T7NVM/GcW4S2ZIiIuOzuO9KZdgQKyXEzS7WVWthGyuJ49jfJ8vpDs7j9hayBLfWoIHneliGW1yJmHW7jwJ901UoFIohQh+6CoVCMUQcSV4olEHVv/jK/zC+y5cgPWTzoEqzJ0AjfuRlFLwZp6Iefs+0n3kkQ7D9pN6ATcSndjsR22gCs1CK6x9ch7ZDNLxPNUe3Kctnv0nthUSkOop9dogiZ6mNzzNPPy4PQzZvFjTx5eBMrPlTkAEiomZvXoYFaXwUlFxE5GwVskd6AbalFSoQMjkF+jmZxrXqdMl7IyK9Do6/1yIqvIdzmhnDefgqtWyaTNDtLbpe2SLGyjh14B3fwTKVqpkNNAj/8Gf/fhyz3W7hDGyIzzwBO9UqXd/pcZwrEfMY+zQmuxRn0rAz5Ss49w2yLYqIXLsMS2HOpQyvCq59NotjjNKgzvWaua6VRUh3bRf34OknXsDnPdwbtbuQTda6hyt4s7oGCaXd4XrNuI5Zso8lGlonihVhrPYoD2x2HnLZwvwFWj5RaIm8fEtLkAHevYtzygVrWvX1OG7uQyoQEdltkk3MxnbaJCl2G4jPjmN/81nTunfoFtUDoG+6CoVCMUToQ1ehUCiGiCPJC9wNd/TUqPEd18S0HLz6N4k+2jYVvCDHQbsDaiQisk3dV5n6c4fViOkrzZC6bXNdmTwVVKFWHOwgcKn1x9g4soSaTZNuLy3B8RAQnStacGuMVh9eYGRlfd34O9fA5Sjm4Eb4zrfhfvjI8x+P4zRlVa3RzLaIiOOAot/awUz9nXXQtItVzA5/5hgyx4o983jrezjGYA+z2F4N67qSwr5PlSBn7AamNNPbx+/PzWN2vurCpfDmLrVmuv3wzD4RUwI5dhztiU5Mgia+9i3UoN2oUX3WRO1VzoTiSit9mo0/cQoujmoVckQ3ce46Dhd3AU1lIm2TVFAjKWtzf1UYzR72pVXHWPM72KZXJhcHSTk7G6ZDYxC6dA9y8Zpshmvj4jxYkWlfCLkyTQry0cjMefyeitHcuAGpwDdL0chOG/eARfWWb925G8drSzfieHMN9+WJ83ASiYgcPw158/4OnkUfPfdsHC/WIUncvQuXTco1nyXTBTNz7v1C33QVCoViiNCHrkKhUAwRR5IXciXQr3SiGITRNoaohp/CzODd26jHOz2FmeNc3kxo6HbxWr+8Ciqep9q4GUpi4P8gmbRJe9pUH9OnJbfJyF8kY/rkBPbLtU36WWuBAl5/8/U4vnAK+58lY/ogbO+aBu6dPcgb+TyoaJ1mXi88Bmr02NNozfKff+3fGesaKYOqpT2q09vDepc3QF/f+TgSB9JjphPixL//jTjObKIucrtHM7+ncB3yWdDKWtakjE2i8vk0KP5iGxT5t2pY7zlqO/QgjFThBjgxg+3vbOEYG33QwiCkGsHthARCRYV4DLMU1iO5bHsX1+fMCfN+WG9AClu/j9q6FVrX3dsoRuNR/Rgv0XKq1SanDUkd198ARX/qsUfi+PZdFDRq9g/3fsUjPaRz1Kdxk85COitWzcJSuTLGfToPmandxb7Xd+lezuE+W1/F2BIRGTsJZ0OYg5SWXUdSUFmw3gY9C2o1nGsRkQK1asqmcJQNkoPyGdx/x+cwhrh1l4iILwcnkBwW+qarUCgUQ4Q+dBUKhWKIOJK8kKJZfisyn9vcSsMhqaHdx6z1a9/9WhyfOg7D+ukF1LwVEclTDv0i1eZNk0OiVKI6mWTwz+fNOg6BD8pYpxniV7/69Tg+M48Z9BTV3LQTZuiNHcxWX7sEeeHcMTgLwujhBuoXP/zD5gcu6G+PavZOtUGVivcxQ1sPcH7nPLOzcC0EHWRTP+9VliiqQ8Z7f9OkeVWqZZrhmgMVzOA/VcL5upyjnPlzqAUgItLYAjXboySVXhfX5MkF1NxY2zTbQQ3CxSfIZN/DWNvYxXlokAuFk208z6wJ26XWQSlyZUR0Ttk141Ldix2q2SsisrGBv0eLkBq6LUgofXLdcCqInWjn1COJLFWFQ6NOLWvSVJ96m5wqLrVQeiBoXywXks3YPNwATzwJyWf22Kzx85NTOJfNBq7p1h7iu8skNa7CoTRXxDGJmMkVi3fQhmh7C4kemVlIbKcm8fl23XSRbK3CBZOmc/EmSYWfeBbn9+kPQSb5zd+AQ0JEZK9jjpf3C33TVSgUiiFCH7oKhUIxRBxJXhgpg7q7lrmKZgDHQUjG6RQt1+uDXiytgtZmsqbpuLAPKtpqgqYtrsPs36HPU+SkyJfM5ASXqHutBmN5ykN88zZovGOBpl24YM6gv335zTger1ApvlHMxG7VHz7D+S8/+0vG37VFUKCNn/9ncbwXga5/7rd/J467EahZd8SUU+wpzL7yrDu3v0mTBHL2m5gBLyakiuqjqF9QrID6y0dQ0tD+KvZrbw8m83euXDPW9cmf+xn8Qdf7vc+jPOHFcchK3RZM6g/C+CTO/d2bSBTpkVsim6f2QiRF7WyZNQ4si2QE0mMmyNFSos7CU3sYgzdrZv2QkBIiXA/Hu0sShpNK0/KguI5j3lslqpsR7MOZkEljvZf+gLpuk4En3Tdn4AfBK2CsrdepI/Z1JFcsbuH+nRkza2NU6X5geXFjF+vqUl2UdhvHmEmbtUi4BOMaORvOn0VrpuNjoP6jo3DKvPOuebxf/ybuLWcCjosKlYz82pdxX3/jVZQqbbXMa3rh7Lx8EOibrkKhUAwR+tBVKBSKIUIfugqFQjFEHEnTDdg6Y5mZXx5pt9yqOZeC9hN6+E2HCllUq2YmVDYDrWpnC1lsPuleobDtCL8Na2bLmXQW+1UZg8VnOg+Nsr6PrJWAxLxczsyUC/vQ8F54DrVB9ynr5v69e/Iw5BLZWquX3o5jj7qYZ6l+8CeryDh6owdd7/WOWT8014ZmGNK/1h61AWpS3dltyjiq9BNFTLbx916Ifam8gUyqz1/H9XEuQgP+ib/3aWNdPcom/Nmf+Wks9+KLcfx3P/034/ipN6/G8Re+9CUZhGtvXYnjfB7jLl/E9ioV6HfcYttxzbkEN8LvUy7VgZ2E5tih8XWPrk921qzNWylh7GxSBmJE94lDOnu9iXWlEsVq3S6scEL1bdv7uHb7pJd2uvjcSR/O5rS+T8eeoRZQAQZkfeNuHF9dMa2REVlId9dh96uO4jo02nT/h1S8KlnHuYf7bJbqbjevo+3RVgfnpN/GdTw+ad5bYQ/73yF74toixnB3G8tYNu5lOzLr6e5srskHgb7pKhQKxRChD12FQqEYIo4kL/g2W2pMCpSxqO4mMY8G0RP+RY8yaBZXTEp+ljLUSpNUW1MoJgnDIWrTSrT+iXxQGtvDYVt0CtjSUyyADnlp8zQ9/igsJwWSRH7vzT+M4z7VMh2E/R2znu6VNVjhvnIF9pWFWVCwR0qgiTNEJY/tmzTvyhascCPjoGbje6BwF5uQF75OrWR2K2Zn4XAddp3cLXR/niHb03oJ63pihCxft2EJFBH5yV/6J3HcrMOqdfofwEo2OgK728degC3tQXDJLjg+gWvS7XDrHRprizjXlm1eXzcL+93EMUglo9M4j8v7yJCq5TDu3JZJkVtU53e3hu+KI5AdHMrMs+nu4I7OIiKb+1Qfdwfn7ux5WJh6VO+YuzWfnSWrn4hsbJhy1A/gCdN17Ltj494oULue/IjZYsdz8V2zjPv3/gaK1HTpOrTr+LzXMbP5+pSBd58scrtNnJf5Pp43xzuIs7lEkRqSY4Iajt0KyLpHFrc+Wf2mxk3Zc3xyUj4I9E1XoVAohgh96CoUCsUQcbSCN33qHBuYM3utiNrqEG1rU8EZlhciylrbd8zMII9oXpsoiU/1TmkCXmz6H2In/p9ErHUQ9Q+pQE9EmVsra5jVHCuYdPvEDGaoN2rYTrkEWr2+uyIPg5eoRezPgya2fhS0+g4VV9mg9jdegP2tZkyHxSMRTkyP5JEVm7LFIsyUrxDN+ub+XWNdrS3qiEt0+S+fRcfj56mYy7cuXYrjz/72F4x1VWdQMOe/fP4/xfGxGRROaTSIoh+y8eqpBRS8kRD7OzkJKahJdDVyqebuttkpN5UBZX786efjuL0NScIjKapEO7ndMGUlvjsCGnftOtwPPer8PDuL81ApmtleGcoUtChbbX8f58siijw3hy7Sdvrh7aNEROYmcc9x/d6NXezv5ioyxfxRnEcRM7tudxdSxeoyyQs+9jEKcU6C0JQqI7q5G3Sf2lSAaWkZZ/jefYzhIDQdONy22KY60BG1AYuocBEXxeGiRyIi91fNWr3vF/qmq1AoFEOEPnQVCoViiDiSvDBPtTw3OiY1C0guKHugKmkrfeAyQtTK75m7k6ZupKMVUPcWUTOu58szjkGiS6lLNXh7EdWXpeXcFNaVz2Mm1CKHhYjIpbdRhOX8xRfiuNJHsZA799+Th8FPdKGtrYGCTRRBdcrkBiiS2T6fBQ0uZswiNYU0qGm/hVnhLepe3GGHR8D1d02KXGtC9lndotq+11BE5HweNPM3N+FC6RTN2e1/88u/EMczk5Aa2k2z4+oPEB2iLrGIyO98HddkvARaWyG3x8goF1QhF8i0WRN2tAr5qEKdr9+6+v047tL+Bi7Glu+bcluHkk64lVWbqHue5J/9HdxPzbo5m7+4iPHhZXB9mw24FLgb9/Y2lnfzZrLQIKRcnJcyyV82JUGVMqD6WztmF+pdSjCKSEhsNZAYUtvHcRWKeEZEKbNoE2uH/S7Wyy4Ddk+5JAPkUmYySIoSYHg5h6QKlhG8LM5vLmMmWqRS2g1YoVAo/sxAH7oKhUIxRBytnm4RXT4TE44GxS95oJZZMk1HPJNIlH5t3WwTc/sGahEcL6F9xm0ybbN5okr7xU4EEZEeLZhxsC8BtSfJWCRnFEGX76+bRnLfg9xw7QZM8pNjkBcqRdNNcBDCwNzH7V1Q98/+x/8Qx2fI1bC1g31p0XnIF8x2LPk0jrFA1CqgmqwezYZnDTqVkCroWEr0m4oNyrlJDou5OTgGPv3Xf8JY1/wcpKnGvkmffwDj2iXqDwxCfw/U3y+hRdAOGeYXL2M8pYmecysnEbNetMMtc0iKssjh0KOaAelEIk2H6nGEJCeNUQJKhqSg23chCZQSrhmHEoE4waBAy3l0fY/Nn4rjMxeeMtb1hc/9qhyEDFHpkK5DuYIxVKng8xkz50J6XZbuSDoMMD46HeoyTEkLXd+81vwbl2QAlgSM7uP086QzyE2zVEIJVQ4ltpA049hYJqlwhTTWv/O/5X1D33QVCoViiNCHrkKhUAwR+tBVKBSKIeJImu5bt9GXaL1u6p0RZYJkUtCHpiooGsG9nzpdWG+aiQIflXEIRstkp1qnYh1pjzSoPrSiyVGzrmm7je24lK0VUKbLtg+NcZuscCdHzNbwGw1kJjU7KCzTzEGbc72H1y/tJoqjPHb+fByPUqbP0jKKfdisNVEmVC9h8+p0cLzrPhcowvXha/UgY5ZH5/iRM9BLpylj8NzjyAj7yRfRFnthwewn1ekeXAiIdVxTjg//yLIHYe5xsu6NQf+vba7G8R7p/x/9xE/FcTlRx7m+CxvUzjoK9ty5+noc95qwQBXyGE9jiQIw9V2MyVwe2ivb17j/l0NaZGiZ70TzZ1B859pbKDzUp+y4j/7wj8bxwiOk49qm7WkQLHoP8zLUuy1ka2XK+IWBAi6eRdppkMhcPWi9VmIUsv2OM8x4vTyGWWs1tF4x7WBc/IbvB7bbOQM0ZBGR8JA2xkHQN12FQqEYIvShq1AoFEPEkeSFt6kdczcws7X4dZ/brt+10OKCW7PbRKHGysi8EhEZG4NEcPU66NTqImVFkXXnvo/6tDfToOQiIlmi+6NkgUq72H7gQ97IUj3dctG0Yy2vk1Th4TubsuO2a2am3kH4xX/6r4y/L11Gy5mUYbGDDNGloikWnbtu18zoGmS04s8ttkNZHJq/5gygJhW8KT75dBxvTUIO+SGyhfVJvvm/+3ywFYftRUzeDpuR9siHIC9sruLad9s4d9kyZATLxfXt9xItZ0LQdT/AGO7T540WxnAhj2VmZpFlJyKyVcNvWg1kyr37NrL5Om0uWIN1RYmiLXvU6v3Pf+pTcXz85Nk4rozAthhSDdxkzeBBcFIHb5/lH64V7STWy3Sdr3XSShcvQ3EQmGMlRXZS12iphF/xKQqMZ5H5PslyAxcFYqmAl2G5zkpIFdwq6SjQN12FQqEYIvShq1AoFEPEkeSFiSlQyVbXdBz0qAstZ5T0qfBH2CPaQvxibhr1P0VEAnIj5LKULVY6ONuLaXGY6GjrB/j7fhMyhE+dPr0MTscz08jmSSVmL7tUKMYLIVtkie6nQvM3B+GdO6YEks7BDXDyFBwTa2v4nJlORNl0/b4p87SpVUufaoaGdE042YuzdPzETDMXZ1lehnPj+9uQjD588uU4dh2m5wl5wZALBmX94HPrkBlpDhc7ojZGo8dAvX1yTly/9N04bu2b9VFb+3CkdNuQBLiOq01ZZD65YRqJerq9NiQBS3AuvAx+k83hfsrkca1PLGDfRUSefvajcXxyHl2hLYccPCRxWeQect3DuReu34LsEYUH02iWBDlLTkQkGOAMcI1uxIOu7+BrzTIASwrs4OFfO4kauD7dH7xf7Hjg7EPLyFoz7+UPaF7QN12FQqEYJvShq1AoFEPEkeSFqRG0/mj1TXkhMF7daUaaJIVWi7qaNlDnc2vdrM2Zz8JM3iGal1WzJm8AAAMWSURBVBsFVbKJ3mQD0NqUmJRgJA1JYqeOda3vwWUwUQDNmxwh58RlFLUREfEcUCU3jX3Zb4JKOskZzwPwr//5Lxh/t6mATYuSG27fgUGfW7bw7G6rZboX/uv//F9xvLIKGaBMxVy2t0GjjU6oCanizJmTcfzSi2hf8+c+BsfA9DhmzXskZ9gJamYQSKKpzDKjAfTxgaBxMHsMDoLNdSRHbNeQhNBpYtz5fbNzrEvFcFJpkkqoFrFP477dw7ZXN8x1ZfNwSVQqiEfHaKxNwe1RnUTsZc0WO5aNcbeyjGvqkHQQ0XsUj83oAdSd8eVXvnGo5RRHh77pKhQKxRChD12FQqEYIqwHmc8tyzrwy1/8x3+LV2B8xzViU6ReOESFeyHRV5Ig3NCcYY1o83vUwdOj2XGX6ocGRp5+YlaV+rKyk6Ef4nO3i30sEA3f7ZqtTtJdMslTTdiFc2fieKcBKvsr/+LzchBuv33V+NswcBP1NuotDLhedWrZIiKytokZeY9mjtMersnG5sHyQrIdycI8KG8hB8mHZQjD2E77njT4W/aA2Wo6rEG57SfOnj3wcxGzHYuZdBEe+PmhwUkbR1gByyY8I86Ge9s+2OkSJVpOycHGj0N1TE4uEvj9A5dT/PEg4j5CCeibrkKhUAwR+tBVKBSKIeJI7oVuSHnJiZYzaR+rbPigvJkcKC6LFh51ZfUd839AM4LLQFxQrQ7lRQeUhJAmk3jkmtQsTfTTdkCF05Rb305hRnqzD+qddk26HbjU4oe6Bq/uI9khn8rKw/B3PvPTxt+2UQBhEDs5uHTeH6mXwO1K2FhO594ZJFskts1l+cw2SA+fEU9ScoNuczLLIOp+SEbv9/900uVB9SXY4c/Sl+L/feibrkKhUAwR+tBVKBSKIUIfugqFQjFEHMkyplAoFIrBUMuYQqFQ/CmBPnQVCoViiNCHrkKhUAwR+tBVKBSKIUIfugqFQjFEPNC9oFAoFIo/XuibrkKhUAwR+tBVKBSKIUIfugqFQjFE6ENXoVAohgh96CoUCsUQoQ9dhUKhGCL+D9HXdZjGfBpzAAAAAElFTkSuQmCC\n",
+ "text/plain": [
+ "