lectures/makemore/makemore_part5_cnn1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## makemore: part 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt # for making figures\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32033\n",
      "15\n",
      "['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia']\n"
     ]
    }
   ],
   "source": [
    "# read in all the words\n",
    "words = open('names.txt', 'r').read().splitlines()\n",
    "print(len(words))\n",
    "print(max(len(w) for w in words))\n",
    "print(words[:8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n",
      "27\n"
     ]
    }
   ],
   "source": [
    "# build the vocabulary of characters and mappings to/from integers\n",
    "chars = sorted(list(set(''.join(words))))\n",
    "stoi = {s:i+1 for i,s in enumerate(chars)}\n",
    "stoi['.'] = 0\n",
    "itos = {i:s for s,i in stoi.items()}\n",
    "vocab_size = len(itos)\n",
    "print(itos)\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle up the words\n",
    "import random\n",
    "random.seed(42)\n",
    "random.shuffle(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([182625, 8]) torch.Size([182625])\n",
      "torch.Size([22655, 8]) torch.Size([22655])\n",
      "torch.Size([22866, 8]) torch.Size([22866])\n"
     ]
    }
   ],
   "source": [
    "# build the dataset\n",
    "block_size = 8 # context length: how many characters do we take to predict the next one?\n",
    "\n",
    "def build_dataset(words):  \n",
    "  X, Y = [], []\n",
    "  \n",
    "  for w in words:\n",
    "    context = [0] * block_size\n",
    "    for ch in w + '.':\n",
    "      ix = stoi[ch]\n",
    "      X.append(context)\n",
    "      Y.append(ix)\n",
    "      context = context[1:] + [ix] # crop and append\n",
    "\n",
    "  X = torch.tensor(X)\n",
    "  Y = torch.tensor(Y)\n",
    "  print(X.shape, Y.shape)\n",
    "  return X, Y\n",
    "\n",
    "n1 = int(0.8*len(words))\n",
    "n2 = int(0.9*len(words))\n",
    "Xtr,  Ytr  = build_dataset(words[:n1])     # 80%\n",
    "Xdev, Ydev = build_dataset(words[n1:n2])   # 10%\n",
    "Xte,  Yte  = build_dataset(words[n2:])     # 10%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........ --> y\n",
      ".......y --> u\n",
      "......yu --> h\n",
      ".....yuh --> e\n",
      "....yuhe --> n\n",
      "...yuhen --> g\n",
      "..yuheng --> .\n",
      "........ --> d\n",
      ".......d --> i\n",
      "......di --> o\n",
      ".....dio --> n\n",
      "....dion --> d\n",
      "...diond --> r\n",
      "..diondr --> e\n",
      ".diondre --> .\n",
      "........ --> x\n",
      ".......x --> a\n",
      "......xa --> v\n",
      ".....xav --> i\n",
      "....xavi --> e\n"
     ]
    }
   ],
   "source": [
    "for x,y in zip(Xtr[:20], Ytr[:20]):\n",
    "  print(''.join(itos[ix.item()] for ix in x), '-->', itos[y.item()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Near copy paste of the layers we have developed in Part 3\n",
    "\n",
    "# -----------------------------------------------------------------------------------------------\n",
    "class Linear:\n",
    "  \n",
    "  def __init__(self, fan_in, fan_out, bias=True):\n",
    "    self.weight = torch.randn((fan_in, fan_out)) / fan_in**0.5 # note: kaiming init\n",
    "    self.bias = torch.zeros(fan_out) if bias else None\n",
    "  \n",
    "  def __call__(self, x):\n",
    "    self.out = x @ self.weight\n",
    "    if self.bias is not None:\n",
    "      self.out += self.bias\n",
    "    return self.out\n",
    "  \n",
    "  def parameters(self):\n",
    "    return [self.weight] + ([] if self.bias is None else [self.bias])\n",
    "\n",
    "# -----------------------------------------------------------------------------------------------\n",
    "class BatchNorm1d:\n",
    "  \n",
    "  def __init__(self, dim, eps=1e-5, momentum=0.1):\n",
    "    self.eps = eps\n",
    "    self.momentum = momentum\n",
    "    self.training = True\n",
    "    # parameters (trained with backprop)\n",
    "    self.gamma = torch.ones(dim)\n",
    "    self.beta = torch.zeros(dim)\n",
    "    # buffers (trained with a running 'momentum update')\n",
    "    self.running_mean = torch.zeros(dim)\n",
    "    self.running_var = torch.ones(dim)\n",
    "  \n",
    "  def __call__(self, x):\n",
    "    # calculate the forward pass\n",
    "    if self.training:\n",
    "      if x.ndim == 2:\n",
    "        dim = 0\n",
    "      elif x.ndim == 3:\n",
    "        dim = (0,1)\n",
    "      xmean = x.mean(dim, keepdim=True) # batch mean\n",
    "      xvar = x.var(dim, keepdim=True) # batch variance\n",
    "    else:\n",
    "      xmean = self.running_mean\n",
    "      xvar = self.running_var\n",
    "    xhat = (x - xmean) / torch.sqrt(xvar + self.eps) # normalize to unit variance\n",
    "    self.out = self.gamma * xhat + self.beta\n",
    "    # update the buffers\n",
    "    if self.training:\n",
    "      with torch.no_grad():\n",
    "        self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * xmean\n",
    "        self.running_var = (1 - self.momentum) * self.running_var + self.momentum * xvar\n",
    "    return self.out\n",
    "  \n",
    "  def parameters(self):\n",
    "    return [self.gamma, self.beta]\n",
    "\n",
    "# -----------------------------------------------------------------------------------------------\n",
    "class Tanh:\n",
    "  def __call__(self, x):\n",
    "    self.out = torch.tanh(x)\n",
    "    return self.out\n",
    "  def parameters(self):\n",
    "    return []\n",
    "\n",
    "# -----------------------------------------------------------------------------------------------\n",
    "class Embedding:\n",
    "  \n",
    "  def __init__(self, num_embeddings, embedding_dim):\n",
    "    self.weight = torch.randn((num_embeddings, embedding_dim))\n",
    "    \n",
    "  def __call__(self, IX):\n",
    "    self.out = self.weight[IX]\n",
    "    return self.out\n",
    "  \n",
    "  def parameters(self):\n",
    "    return [self.weight]\n",
    "\n",
    "# -----------------------------------------------------------------------------------------------\n",
    "class FlattenConsecutive:\n",
    "  \n",
    "  def __init__(self, n):\n",
    "    self.n = n\n",
    "    \n",
    "  def __call__(self, x):\n",
    "    B, T, C = x.shape\n",
    "    x = x.view(B, T//self.n, C*self.n)\n",
    "    if x.shape[1] == 1:\n",
    "      x = x.squeeze(1)\n",
    "    self.out = x\n",
    "    return self.out\n",
    "  \n",
    "  def parameters(self):\n",
    "    return []\n",
    "\n",
    "# -----------------------------------------------------------------------------------------------\n",
    "class Sequential:\n",
    "  \n",
    "  def __init__(self, layers):\n",
    "    self.layers = layers\n",
    "  \n",
    "  def __call__(self, x):\n",
    "    for layer in self.layers:\n",
    "      x = layer(x)\n",
    "    self.out = x\n",
    "    return self.out\n",
    "  \n",
    "  def parameters(self):\n",
    "    # get parameters of all layers and stretch them out into one list\n",
    "    return [p for layer in self.layers for p in layer.parameters()]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(42); # seed rng for reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "76579\n"
     ]
    }
   ],
   "source": [
    "# original network\n",
    "# n_embd = 10 # the dimensionality of the character embedding vectors\n",
    "# n_hidden = 300 # the number of neurons in the hidden layer of the MLP\n",
    "# model = Sequential([\n",
    "#   Embedding(vocab_size, n_embd),\n",
    "#   FlattenConsecutive(8), Linear(n_embd * 8, n_hidden, bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "#   Linear(n_hidden, vocab_size),\n",
    "# ])\n",
    "\n",
    "# hierarchical network\n",
    "n_embd = 24 # the dimensionality of the character embedding vectors\n",
    "n_hidden = 128 # the number of neurons in the hidden layer of the MLP\n",
    "model = Sequential([\n",
    "  Embedding(vocab_size, n_embd),\n",
    "  FlattenConsecutive(2), Linear(n_embd * 2, n_hidden, bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "  FlattenConsecutive(2), Linear(n_hidden*2, n_hidden, bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "  FlattenConsecutive(2), Linear(n_hidden*2, n_hidden, bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "  Linear(n_hidden, vocab_size),\n",
    "])\n",
    "\n",
    "# parameter init\n",
    "with torch.no_grad():\n",
    "  model.layers[-1].weight *= 0.1 # last layer make less confident\n",
    "\n",
    "parameters = model.parameters()\n",
    "print(sum(p.nelement() for p in parameters)) # number of parameters in total\n",
    "for p in parameters:\n",
    "  p.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      0/ 200000: 3.3167\n",
      "  10000/ 200000: 2.0576\n",
      "  20000/ 200000: 2.0723\n",
      "  30000/ 200000: 2.5134\n",
      "  40000/ 200000: 2.1476\n",
      "  50000/ 200000: 1.7836\n",
      "  60000/ 200000: 2.2592\n",
      "  70000/ 200000: 1.9331\n",
      "  80000/ 200000: 1.6875\n",
      "  90000/ 200000: 2.0395\n",
      " 100000/ 200000: 1.7736\n",
      " 110000/ 200000: 1.9570\n",
      " 120000/ 200000: 1.7465\n",
      " 130000/ 200000: 1.8126\n",
      " 140000/ 200000: 1.7406\n",
      " 150000/ 200000: 1.7466\n",
      " 160000/ 200000: 1.8806\n",
      " 170000/ 200000: 1.6266\n",
      " 180000/ 200000: 1.6476\n",
      " 190000/ 200000: 1.8555\n"
     ]
    }
   ],
   "source": [
    "# same optimization as last time\n",
    "max_steps = 200000\n",
    "batch_size = 32\n",
    "lossi = []\n",
    "\n",
    "for i in range(max_steps):\n",
    "  \n",
    "  # minibatch construct\n",
    "  ix = torch.randint(0, Xtr.shape[0], (batch_size,))\n",
    "  Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y\n",
    "  \n",
    "  # forward pass\n",
    "  logits = model(Xb)\n",
    "  loss = F.cross_entropy(logits, Yb) # loss function\n",
    "  \n",
    "  # backward pass\n",
    "  for p in parameters:\n",
    "    p.grad = None\n",
    "  loss.backward()\n",
    "  \n",
    "  # update: simple SGD\n",
    "  lr = 0.1 if i < 150000 else 0.01 # step learning rate decay\n",
    "  for p in parameters:\n",
    "    p.data += -lr * p.grad\n",
    "\n",
    "  # track stats\n",
    "  if i % 10000 == 0: # print every once in a while\n",
    "    print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}')\n",
    "  lossi.append(loss.log10().item())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fb5a03e3b50>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0n0lEQVR4nO3deXxU1f3/8ddnJnvInhBCFpIAASGEgGFVcMEF3IBqcau2amvtV1trF6u168/altbWblZqXWqrdce6L4gLIGuAsIYlLNnJvu+TOb8/ZggTSGDQJBMmn+fjwcOZM/fOfO7N+M7JufeeK8YYlFJKeS+LpwtQSinVvzTolVLKy2nQK6WUl9OgV0opL6dBr5RSXs7H0wX0JDo62iQnJ3u6DKWUOmNs3ry50hgT09NrgzLok5OTyc7O9nQZSil1xhCR/N5e06EbpZTychr0Sinl5TTolVLKy2nQK6WUl3Mr6EVkvojsFZE8Ebmvh9cXish2EckRkWwROdfltXtEZJeI7BSR50UkoC83QCml1MmdMuhFxAo8CiwAJgDXi8iE4xZbCUw2xmQCtwJPONeNB74DZBlj0gErcF2fVa+UUuqU3OnRTwfyjDEHjTHtwAvAQtcFjDGN5tg0mMGA65SYPkCgiPgAQUDJFy9bKaWUu9wJ+nig0OV5kbOtGxFZLCJ7gLdx9OoxxhQDDwMFQClQZ4z5oKcPEZHbncM+2RUVFae3FU5/WbmfT/d9vnWVUspbuRP00kPbCZPYG2NeM8aMBxYBDwKISASO3n8KMBIIFpGv9PQhxpjHjTFZxpismJgeL+46pX98eoBVGvRKKdWNO0FfBCS6PE/gJMMvxphVwGgRiQYuAg4ZYyqMMR3AcmD2F6j3pAL9fGhut/XX2yul1BnJnaDfBIwVkRQR8cNxMPUN1wVEZIyIiPPxVMAPqMIxZDNTRIKcr88DcvtyA1wF+Vlpbu/sr7dXSqkz0innujHG2ETkLuB9HGfNPGWM2SUidzhfXwZcDdwsIh1AC3Ct8+DsBhF5BdgC2ICtwOP9syka9Eop1RO3JjUzxrwDvHNc2zKXx0uBpb2s+3Pg51+gRrcF+llp0aBXSqluvOrKWEePXsfolVLKlVcFfaCvjw7dKKXUcbwq6IP8rLR0aNArpZQrrwt67dErpVR3XhX0ejBWKaVO5FVBH+y8YOrYtDtKKaW8KugD/azYDbTZ7J4uRSmlBg2vCvogPyuADt8opZQLrwz6Zj3zRimlunhV0Af6OS70bdGLppRSqotXBX2Qr7NHr0M3SinVxbuC3k+DXimljudVQR/YFfQ6dKOUUkd5VdAHOcfotUevlFLHeFnQ69CNUkodz6uCPlDPo1dKqRN4VdBrj14ppU7kVUEf4HO0R68HY5VS6iivCnqLRQj01amKlVLKlVcFPUCwv1WnQFBKKRdeF/Q6J71SSnXnVtCLyHwR2SsieSJyXw+vLxSR7SKSIyLZInKuy2vhIvKKiOwRkVwRmdWXG3C8IF8fvWBKKaVc+JxqARGxAo8CFwNFwCYRecMYs9tlsZXAG8YYIyIZwEvAeOdrfwbeM8ZcIyJ+QFCfbsFxAvV2gkop1Y07PfrpQJ4x5qAxph14AVjouoAxptEcu61TMGAARCQUmAs86Vyu3RhT20e19yhIh26UUqobd4I+Hih0eV7kbOtGRBaLyB7gbeBWZ3MqUAE8LSJbReQJEQnu6UNE5HbnsE92RUXFaW2EK71BuFJKdedO0EsPbSfclNUY85oxZjywCHjQ2ewDTAUeM8ZMAZqAE8b4nes/bozJMsZkxcTEuFN7jwL9fGjRs26UUqqLO0FfBCS6PE8ASnpb2BizChgtItHOdYuMMRucL7+CI/j7TZCvVQ/GKqWUC3eCfhMwVkRSnAdTrwPecF1ARMaIiDgfTwX8gCpjzBGgUETGORedB7gexO1zgX5Wmtu0R6+UUked8qwbY4xNRO4C3geswFPGmF0icofz9WXA1cDNItIBtADXuhyc/TbwnPOXxEHgln7Yji5Bfo4LpowxOH/3KKXUkHbKoAcwxrwDvHNc2zKXx0uBpb2smwNkff4ST0+Qn5VOu6G9046/c+4bpZQayrzuythgf8fvrsZWHadXSinwwqCPCwsEoKS21cOVKKXU4OB1QZ8U6bjwtrCm2cOVKKXU4OB1QZ8Y6ejRF1Rr0CulFHhh0IcE+BIR5KtBr5RSTl4X9OAYvinUoFdKKcBLgz5Bg14ppbp4ZdAnRQZRVNNCp/2EKXmUUmrI8dqgt9kNpXUtni5FKaU8ziuDPjHCeYpltQa9Ukp5ZdB3nUuv4/RKKeWdQR8XHoDVInqKpVJK4aVB72u1kBQZxL6yBk+XopRSHueVQQ+QkRDGtqJaT5ehlFIe57VBn5kYTll9G0fqdHIzpdTQ5rVBPzkxHICcwlqP1qGUUp7mtUE/IS4UX6to0CulhjyvDfoAXysT4kLZpkGvlBrivDbowTF8s72oVqdCUEoNaV4d9FnJkTS1d7Imr9LTpSillMd4ddBfOjGWkWEB/GXlfozRXr1Samjy6qD397HyrQvGsDm/hs/yqjxdjlJKeYRbQS8i80Vkr4jkich9Pby+UES2i0iOiGSLyLnHvW4Vka0i8lZfFe6uJVkJDA/x59/rDg/0Ryul1KBwyqAXESvwKLAAmABcLyITjltsJTDZGJMJ3Ao8cdzrdwO5X7jaz8Hfx8qcsTFk59fo8I1Sakhyp0c/Hcgzxhw0xrQDLwALXRcwxjSaYykaDHQlqogkAJdzYvgPmGnJEVQ3tXOwsslTJSillMe4E/TxQKHL8yJnWzcislhE9gBv4+jVH/Un4F7AfrIPEZHbncM+2RUVFW6U5b6s5AgANh+u6dP3VUqpM4E7QS89tJ0wBmKMec0YMx5YBDwIICJXAOXGmM2n+hBjzOPGmCxjTFZMTIwbZbkvNXoY4UG+ZOdX9+n7KqXUmcCdoC8CEl2eJwAlvS1sjFkFjBaRaOAc4CoROYxjyOdCEXn285f7+VgsQtaoCLK1R6+UGoLcCfpNwFgRSRERP+A64A3XBURkjIiI8/FUwA+oMsbcb4xJMMYkO9f7yBjzlT7dAjedPSqSg5VNVDW2eeLjlVLKY04Z9MYYG3AX8D6OM2deMsbsEpE7ROQO52JXAztFJAfHGTrXmkF2ikumczbLXSX1ni1EKaUGmI87Cxlj3gHeOa5tmcvjpcDSU7zHJ8Anp11hH0mLHQbAvrIG5qb17TEApZQazLz6ylhXUcP8iQr2Y39Zo6dLUUqpATVkgh5gbOww9pXrfWSVUkPLkAr6tNgQ8soa9QpZpdSQMqSCfmxsCA1tNkr1PrJKqSFkSAV92vBjB2SVUmqoGFpBHxsCoAdklVJDypAK+ohgP6KH+ZOdX41dby+olBoihlTQA8xPj+X9XWUs+cc6Gttsni5HKaX63ZAL+gcXpvPgwolk59fw0Z5yT5ejlFL9bsgFvYhw3fQk/Hws7Ciq9XQ5SinV74Zc0AP4Wi1MiAtle1Gdp0tRSql+NySDHiAjIYydxXV06kFZpZSXG7JBPyk+jKb2Tg5V6qmWSinvNmSDfrJz2mIdvlFKebshG/SjY4YR6GvVoFdKeb0hG/RWizA5MYzXc4pZsbvM0+UopVS/GbJBD/CrRemMCAvkG//OZtW+Ck+Xo5RS/WJIB/2Y4SH8787ZRAb78WJ2oafLUUqpfjGkgx7A38fKlRlxfLi7jPrWDk+Xo5RSfW7IBz3AoinxtNnsvLfziKdLUUqpPqdBD2QmhpMcFcQ/Pj3A5vwaT5ejlFJ9yq2gF5H5IrJXRPJE5L4eXl8oIttFJEdEskXkXGd7ooh8LCK5IrJLRO7u6w3oCyLCT6+YQF1LB1c/tpZ3dpR6uiSllOozpwx6EbECjwILgAnA9SIy4bjFVgKTjTGZwK3AE852G/B9Y8xZwEzgzh7WHRTmnRXLqnsvYMzwYfztozy9r6xSymu406OfDuQZYw4aY9qBF4CFrgsYYxrNsWQMBoyzvdQYs8X5uAHIBeL7qvi+FuTnw+1zUtldWs/aA1UANLR2UFjd7OHKlFLq83Mn6OMB13MPi+ghrEVksYjsAd7G0as//vVkYAqwoacPEZHbncM+2RUVnjunfeGUkUQP8+c37+by0qZCLnlkFZf9eTVttk6P1aSUUl+EO0EvPbSdMK5hjHnNGDMeWAQ82O0NRIYBrwLfNcbU9/QhxpjHjTFZxpismJgYN8rqH/4+Vn5y+VnkVzZz76vbaWy10dBm06kSlFJnLB83likCEl2eJwAlvS1sjFklIqNFJNoYUykivjhC/jljzPIvVu7AWDQlnvnpI9heVMfI8ADOXfoxGw5WkRIdzLbCWuadFevpEpVSym3uBP0mYKyIpADFwHXADa4LiMgY4IAxxojIVMAPqBIRAZ4Eco0xf+zb0vtXgK+V6SmRAIwfEcKGQ9XsLWvkre0l5PzsEsICfT1coVJKueeUQW+MsYnIXcD7gBV4yhizS0TucL6+DLgauFlEOoAW4Fpn6J8L3ATsEJEc51v+2BjzTj9sS7+ZkRLJC5sKsdkNxkBeeQNnj4r0dFlKKeUWd3r0OIP5nePalrk8Xgos7WG9NfQ8xn9GmZ4SxTPr8rue7ytr1KBXSp0x9MpYN8xIjUQEFmaOJNDXyr6yBk+XpJRSbnOrRz/URQ/z55lbpjMpPoxDlU3sL9PbDyqlzhzao3fT3LQYIoL9GDs8RHv0Sqkzigb9aUqLHUZ5Qxub82v444p9dHTaPV2SUkqdlA7dnKa02BAAbnl6I/WtNgC+d3GaJ0tSSqmT0h79aRobOwyA+lYbGQlhPPpxHpvzq2nt6OSXb+7i9ZxiD1eolFLdaY/+NMWHBxIb6s95aTE8cPkELvvzaq57fD3JUcHsL29k4shQFmYO2nnblFJDkPboT5OIsPL75/PbL2UQFujLG3edw4L0OAprmpmREkluaT0NrR3c9d8tXP6X1azNq/R0yUqpIU6D/nMY5u+DxeK4DixqmD9/uX4KO35xKd++cCx2Ax/mlvHOjlL2lTVwwxMbWH+wysMVK6WGMg36PuJrtTAlKRyrRXj4/X3YDbxw+0wignz512eHPV2eUmoI06DvQ8H+PkwcGUpxbQtJkUFMTYpgSVYiK3LLOFLX6unylFJDlAZ9H8tyzoGzIH0EIsINM5LotBte2FTg4cqUUkOVBn0fm5MWjQhckTESgFFRwZwzJoo3t/U6hb9SSvUrDfo+dsG44ay7bx6TEsK62s4ZE82BiiaqGtt4dn0+dz63hfyqJg9WqZQaSjTo+8GIsIBuz2c4b2Cy6XA1j36cx9s7SrnkkVXkFNZ6oDql1FCjQT8AJsWH4+9j4ck1hyita+WHl47DIsJrW4o8XZpSagjQK2MHgJ+PhczEcDYcqsZqEW6ckcTm/Bo+3VcBgN1uus7LV0qpvqY9+gFydPhmZmok4UF+zB0bzeGqZtYeqGT2bz/ipU2FHq5QKeWtNOgHyPSUKAAunTgCgPPGDQfgW89u4Uh9K795N5fc0noW//0zXtvqGNKxddoxxnimYKWU19ChmwEye3QUf7o2kwWTHEGfHBVEUmQQBdXNXJERx9s7Srnqb2vo6DTsL2skPMiPH768jW/MSeWb5432cPVKqTOZ9ugHiMUiLJoSj7+PFXBMjnZ5Rhyp0cH8/prJXDctCV+rhT9dm0lHp51bnt5EZWM77+064uHKlVJnOreCXkTmi8heEckTkft6eH2hiGwXkRwRyRaRc91ddyi799JxrPjeeQT6WXloUTrr7p/HoinxPHD5WaREB3N5Rhw7iupobred8r1K61oGoGKl1JnolEEvIlbgUWABMAG4XkQmHLfYSmCyMSYTuBV44jTWHbJEBKvzbBuLRQgL9AXg5lnJfPyD81mSlYjNbth4qJor/7qGpe/t6bb+0fH71fsrmPWbj1izX6dEVkqdyJ0e/XQgzxhz0BjTDrwALHRdwBjTaI4dNQwGjLvrqt6dPSoCq0V46O1cdhTX8eSaQ5TVOyZHW72/gvN+/wl7jzTw7Pp8AJ7X+XSUUj1wJ+jjAddz/4qcbd2IyGIR2QO8jaNX7/a6zvVvdw77ZFdUVLhTu9cb5u9DenwY+8sbSYoMotNu+MenB2lut3HfqzsoqG7m/uXbWZlbTqCvlRW7yqhr7vB02UqpQcadoO/pSp4TzvkzxrxmjBkPLAIePJ11nes/bozJMsZkxcTEuFHW0HD0/PsfXjqOxVPieXZ9Pl/6+1qKa1tYmDmSLQW12OyG33xpEu2ddt7Y3n3ytM35NXR02j1RulJqkHAn6IuARJfnCUCvUzEaY1YBo0Uk+nTXVSe6aeYovndxGpdNiuNH88dzVeZI2mx2vjk3ld9fM5mU6GCmJ0eyMHMk40eE8N8NBV1j97ml9Vz92FqW61QLSg1p7pxHvwkYKyIpQDFwHXCD6wIiMgY4YIwxIjIV8AOqgNpTratOLjEyiO/MGwtATIg/D395crfXl39rNhaLICJ8Y04q3395Gx/sLuPSiSO6pljYnF/DtdOSBrx2pdTgcMoevTHGBtwFvA/kAi8ZY3aJyB0icodzsauBnSKSg+Msm2uNQ4/r9sN2DFkRwX5dZ+sszBxJSnQwj6zYh91uus7C2VZY17X8q5uLuPVfm/SKW6WGELeujDXGvAO8c1zbMpfHS4Gl7q6r+oeP1cJ35o3hnhe38cKmQjYeribA18K+8gYa22wE+1l59JM8DlY0Ud7QRmzosemU61o62HiomosnxHpwC5RS/UGvjPUyV02OZ0pSOD99fSftNjvXTUvCGNhRVEdOYS0HKxw3PNlRVNdtvec25PONf2dzqFJviKKUt9Gg9zJWi7D06gwsAn5WC7fPTQVgW1Etr24pwt/HggjsLOke9LtL6gHYdKh6wGtWSvUvndTMC6XFhvDLq9Ipb2hlZHggo6KCeD2nhKLqZhakj2BHcR07i+u7rbP3SAMAGw9Xs2RaYk9vq5Q6Q2mP3kvdMCOJ716UBsDUpAhyS+sZHurPnReMIT0+jJ3Fx3r0bbZODjqHbDb20qPfdLiaLQU1/V+4UqrPaY9+CLj/svEsyUpkRkokFoswKT6M13NKqGhoIybEn7zyRjrthszEcHIKaymrbyV6mH/XPDwA9y/fQV1LB6vvvYAAX6sHt0Ypdbq0Rz8EDA8JYNboqK7bFU4cGQbA+7uOUFbfyp5Sx7DNTTNHAfDVpzYy+ZcfdM2I2drRycGKRioa2nh+YwE7i+vYnF+DMYblW4r07lhKDXLaox+CJsaHYrUIP/nfTv7fW7uZPToKPx8Ll2fE8fM3drHf2cP/cHcZN81KJq+8EbuBID8rD7+/l+aOToyB8SNC2HOkAT+rhYsnxBIR7Nfj5328p5x/rzvM4zdn4WvVvoVSA03/rxuCQgN8+e/XZ/C3G6YwzN+HT/ZWkBY7jABfK/+6ZRrv3j2H5KggVu4pBxxTKQD8/MoJdNgNX5kxirvnjeVIfSvXT090zLGzrfeZLZ5Zd5iP91awap9OVqeUJ2iPfoiakeq4h63dwHee38q42FAAspIdk6hdMH44/91QQEt7J3uONBDga+GasxO5emoCPs5e+XcvGouIsK2wjpc3FyICDa027rxgTNfnNLbZWJtXBcDyLcXMO0svyFJqoGnQD3FXZsRRXt/KTGfwH3Xh+OE8/dlh1h6oZM+ResbFhjgPzh47QCviePzlrAR++eZudhbvwmoRbj0nhUA/xwHbT/dW0N5pZ1J8GCtyy6hsbMNuDMNDAlBKDQwduhniRISvz0klPT6sW/v0lEiC/ay8trWY3NIGxo8I7fU9Fk+JZ1pyBFdkxNFpN+wsqcPWaaektoX3dx0hMtiP/7dwIu02OzN/vZJzfvtR13CQUqr/aY9e9cjfx8ot56Twt4/zABgfF9LrsuFBfrx8x2wqGtp4a3spOQW1rMwtZ9mnBwC45uwEMhPDuXFGEp12w4rdZTzw2g5euWN215lASqn+o0GvenXPxWlsK6pl9f7Kk/boj4oJ8Sc+PJCthTVsLahlckIY546N5pqzExERHlo8CYBpyUV8/+VtvJhdyLVZidzyr02kxQ7jgcv1dsJK9QcNetUrq0X42/VT+V9OMdOSI9xaJzMpnBW7ymjvtHPv/HEsnpJwwjJfmhrPS9mF/PbdPbR2dPLpvgrW5FVy08xk6ls7CPC1Mmb4sL7eHKWGLB2jVycVFuTLV2cnd51pcypTEsNp77Tj52Phol7OsHH07tNpbrfxyzd3M2b4MKwW4e4Xt7L4759x05MbaGnv7Fq+sLqZJcvW8dWnNna15Vc18cBrO8grb/xiG6jUEKBBr/rU5MRwAOaOjSEkwLfX5cYMD+Gbc0cD8MurJnLD9CS2FtSSEBFEaV0rT312CIDyhlYu/8tqNh6uZtX+ChpaO3hv5xEueWQVz20o4Nn1+f2+TUqd6XToRvWpSfFhnD0qgq/NTj7lst+/JI0vZyUwKiqY9JFhxIYGcMOMJH7w8jYe++QA109PYt2BKupbbdxzURqPfLiPrQW1PL7qAPERgYQH+rL2QOVJP+P1nGJGxww74awipYYS7dGrPhXga+XVb83m3LHRp1xWRBgVFQw4hoi+df5owgJ9uXveWBrbbHyYW0ZOYS0Bvha+NjsZi8BKZ9sVGSO5ZOII9pU55uABOFjRyDNrD9Npd9wmsaqxje+/tI0/fbiv/zZYqTOA9ujVoDNxZChRwX6sP1BFfnUzk+LDCAvyZfyIUJ7fVIjdwPnjYvBxnpq57mAVc8ZEc/NTGymqaWFzfg1/XDKZN7eVYLMbcgprMcZ0XeDV2Gbj7e0lRAT5cfGE2K52pbyVBr0adESEmaOjWJNXSV1LB19xzqqZlRzB7tJ6IoJ8mZwQDkBIgA9vbSvh2fX5lNe3ceOMJJ7bUIBF6LotYmVjO0U1LSRGBrH+YBXfeCabhjYbABeMi+HP108h9CTHE5Q60+nQjRqUZqVGUd7QRpvN3nWA9+xRjlM856bFYLUIVoswMzWKD3Y7hnN+d00GDy2exA8uSeN/OSVsK6pjYeZIAHIKa6loaOPbz28lJsSfV781m59eMYFV+yu5/9UdGGO6PttuNzy55hBPf3ao2w1alDpTaY9eDUqzRh+beyfT2XuflRpFkJ+VKzJGdr12+9xURoQGcPvcVBIjgwC484IxVDS0sXxrMffOH897O4+wtaCWl7ILaWjt4D+3TWf8iFDOHhVBm62T3723l3M2RnPDjCQA1h+s4sG3dgOOvxg2/+Ri/Hz6pk/U0NrBfct38N15Yxkb2/vVxkr1Jbe+vSIyX0T2ikieiNzXw+s3ish257+1IjLZ5bV7RGSXiOwUkedFRGezUqeUGh3M8BB/IoJ8SYwMBGB4aAA5P7uEiyccOz9/WnIkDy5K7wp5cAz9/HJhOpseuIj48EAmxYfx/MYCVu+v5IHLJ3S7yveOuaOZMzaaB9/aTX6VY6jnvV1HCPC18Mclk2lotbH+YFW32l7YWEDGL95n6Xt72HiomrL6Vre3K/twDW9vL+X/ntvS7VoBoNtfFUr1pVMGvYhYgUeBBcAE4HoROf5a9UPAecaYDOBB4HHnuvHAd4AsY0w6YAWu67vylbcSEW6fm8rXZqd0O1h6Oj3ro7c8zEwMp6Wjk2nJEdw4PanbMhaL8LtrMvCxCPe+sh1bp533dh7h/LThXDYpjiA/Kx/sPtJtnWfW5WOAZZ8eYMk/1jFn6ccU17bQ0NrB6znFJw3sPc6bsOdVNPLQO7u72utaOpj20Ie8s6PU7e1Tyl3u/F8zHcgzxhw0xrQDLwALXRcwxqw1xhy9c/R6wPW6dx8gUER8gCCg9ztUKOXi63NSufuisV/4fS4cP5zoYf785kuTepxELS4skJ9eMYENh6q55V+bKG9oY376CAJ8rZyXFsOK3WXYnads7jlST25pPT+4ZByr772AZV85mw67nec3FPCHD/Zx9ws5ZOcfu4n6ytyybjdc2VfWQFxYAIunxPP61pKu9113oJLKxvZerwtYmVvGuUs/otF5EFmp0+FO0McDrjcFLXK29eY24F0AY0wx8DBQAJQCdcaYD3paSURuF5FsEcmuqNA7Eam+M3tMNJsemMeY4b2PiX85K4E7zhvN6v2V+FqFC88aDsAlE2Mpq29ju/Og7P+2lmC1CJdnxJEQEcT89BHMGz+c5zbk89+NBQB8mFsGOO61+72XtnHPizm0djiGafYeaSAtNoSZKVE0tNk46DwzaPX+yq7Xe7JidxlFNS1sLajp8fVNh6vJKaw9zT2jhgp3gr6nk4x7/NtURC7AEfQ/cj6PwNH7TwFGAsEi8pWe1jXGPG6MyTLGZMXExLhTu1JuO9W58iLCfQvG89DidH40f3zX6ZYXjovFz2ph+ZYibJ123sgpZu7YaKKH+Xet+5WZo6hp7sAYw/gRIazMddyCcWVuOXUtHVQ1tfPa1mJsnXbyKhoZPyKEjETHlbrbi2oBWJPnCPo9Rxq6hn52FNXx0Nu7McZ0hXj24ROD3hjDd1/I4f7lO054raPT3vVLxh25pfV876UcbJ12t9dRg587QV8EJLo8T6CH4RcRyQCeABYaY44evboIOGSMqTDGdADLgdlfrGSl+s+NM0bx9TmpXc/Dgny5IiOOVzcX8Z/1+ZTUtXL9ceP8c8fGkB4fyi3npLAkK5G88kbyq5p4eXMhcWEBTBwZypNrDnGosol2m5202BDGxAwj0NfK9qI6Cqqaya9qZuzwYTS02iitcxzcXbbqAP9cfYj1B6vZV+bo6W92Dgt12g1ffWojS9/bw4GKJoprW9hzpJ665o5utf3g5W0s+cc6t7d/+ZYilm8pJr+6+XPtPzU4uRP0m4CxIpIiIn44Dqa+4bqAiCThCPGbjDGu15sXADNFJEgcXap5QG7flK7UwLh5djJN7Z08+NZuMhLCup31A44Dum99ew4/vuysrhk7f/vuHlbtq+BLU+P5xpxU8sob+d37ewEYNyIEH6uF9PhQx3z/eY6hyq/PSQEcwzdttk4+cd6c/Q8f7MVuYFRUEFsLaui0G57+7BCf7qvgmbWHeW+n4wCuMbDxcHVXXcW1Lby5rYTtRXUcqWvl5exC7n1l20m3dVuhY4iqqKbli+42NYicMuiNMTbgLuB9HCH9kjFml4jcISJ3OBf7GRAF/F1EckQk27nuBuAVYAuww/l5j/f9ZijVfzITw5mcEIbdwPcvGXfSYaCkqCAmxYfx7s4jhAb6cm1WEldOHknWqAhW7C7DInTNtZ+REM6uknqeWH2I5Kgg5qfHAY7hm7UHqmhq7yQ0wKfr4O4tzl84b24r4eEP9pIWO4zm9k7+9nEeiZGB+PlY2OByKuiz6/NxHutlTV4lj31ygJeyiyiqOdZbL69v7ZobyNZpZ0fx0aDXHr03cetcNWPMO8aYNGPMaGPMQ862ZcaYZc7HXzfGRBhjMp3/slzW/bkxZrwxJt0Yc5Mxpq1/NkWp/vOTKybwnXljmevGZG0v3zGLnJ9dzNafXkxSVBBWi/CHJZMJ8rOSHBXcddpnRkIY7TY7h6ua+O3VGYQF+hIXFsDeI/Ws2F1GsJ+Vb1/oOOsoOSqIec6/Fr77Yg7D/H155tbpjI4JprXDzrzxsUxJDGfdwSoefn8v1z2+jmfX53PxhFiigv14as2hrgO/R48hbDhYxezffsTvnX9p7C9vpMU5nq89eu+iUyAo5YZpyZF87+I0tyZAC/C1Eh7k123ZUVHB/PPmLH5+1cSutrNHRWAR+L/zRzMz1XEl8LgRIazJq+Lt7aWcNy6GKyY7evmZieEkRAQyOiaY8SNC+N+ds4kLC+TaaY7DZ+elxTAjNYpdJfX87eM8qpvaCfC18q3zRzNrdBS7S+uxWoT48EA+zC2juLaF/3tuCza74bkN+TS12boODAf4WnoN+m2Ftewu0Ru7n2l0CgSlBsg5Y7r/NZAQEcTqH13IyLBjF4tPS47kk70VpMeHcucFY4gLC+TXiyeRmRiOiPDGXecS4GvF6rwe4OZZyYQH+jE3LYbwIF/++tF+/u/80fzAZYhp35gG3tpeyjljojkrLoSn1hzixn+up91m53fXZHDvK9t5bWsxu0rqCQ3wIT0+jKKaZupbO1i+uYhrshIZ5u9Dc7uNrz29kRFhgbx795yB23HqC5PBeNl1VlaWyc7O9nQZSg04u93Q1G476d25TqamqZ2IYL9ubSW1LZz/8Cf84cuTiQsL4Jpl6xjm78Mzt05nalI4V/5tDdWN7XQaQ1psCCPDAvlobzm3nZvCb9/dw5jhw1j2lbP5ZG85v3rbcS7Fhh/PIzY0gKY2G797bw93XjiG4SE6u4knichm12FzVzp0o9QgYrHI5w554ISQBxgZHsjmn1zElZNHMiUpgu9eNJb/fmMGZ4+KQES4f8FZ+PpYqGnuYN744SRGBlLR0MZHe8qJDfWnuqmdK/66mr9+lEeSc06ho1f7vp5TwjPr8nllcxEA9a0dXdcBbCusPWE+H+UZGvRKDQFHf3lYLcJ3L0ojwzkjKDiGlD794QXsfXA+XzsnhYQIR5hvPFTNgvQ43r17DjNTo6hv7WDp1RnEhPjzqTPoX9nsuGj+kz0VHKpsIutXH/LW9lIKqppZ9PfPeHFTwcBuqOqRjtErpYBjVw8nRAR2tc0aHUVsaABPf20aFQ1tDA8N6Jr/Z39ZA1sKaokJ8WdzQQ2PfZJHu83O6zklVDS0YQwcrtLTNAcD7dErpbo52qMXgZkpUc7HwvBQxxj8eWkx1LV0cP0/N2C1CA8unEin3fBSdhEisHp/BW9td1w8X1yrp2kOBhr0Sqluhof442sVJo4MJSzoxOMFl02K44eXjmNUVBA3zxrFxRNGEBboWO6uC8bQZrOzpaAWcBwIVp6nQzdKqW4sFuGqyfFMHRXe4+tWi3DnBWO484IxXW0L0keQU1jLd+aN5d/r8qlr6WD8iBDt0Q8SGvRKqRP8YcnkUy/k4leL0rHZDb5WCwvSR/Dx3nKuyIjj4Q/20dRmI9hfo8aTdOhGKfWF+VgtXVM7/PzKibz17Tldt3csrdNevadp0Cul+lSgn5WYEH/iwx1n7xTXun9PXdU/NOiVUv1i5NGg1wnSPE6DXinVL4aH+GO1iJ55Mwho0Cul+oWP1cKI0AAN+kFAg14p1W/iwwMpqmlh0+FqGlo7Tr2C6hca9EqpfhMfEcjGw9V8edk6nlxzyNPlDFka9EqpfpOVHMGI0ABCA3y6bnCuBp4GvVKq39w4YxTrfzyPrORIDlY0ebqcIUuDXinV71Kigzlc1YTdPvhudDQUaNArpfpdqvMm5kfq9eIpT3Ar6EVkvojsFZE8Ebmvh9dvFJHtzn9rRWSyy2vhIvKKiOwRkVwRmdWXG6CUGvxSooMBdPjGQ04Z9CJiBR4FFgATgOtFZMJxix0CzjPGZAAPAo+7vPZn4D1jzHhgMpDbF4Urpc4cqdHDADhU2ejhSoYmd3r004E8Y8xBY0w78AKw0HUBY8xaY0yN8+l6IAFAREKBucCTzuXajTG1fVS7UuoMERvqT5CflYOV2qP3BHeCPh4odHle5GzrzW3Au87HqUAF8LSIbBWRJ0QkuKeVROR2EckWkeyKigo3ylJKnSlEhJTo4G5DN5/3xuHGGJZvKaKxzdZX5Xk9d4Jeemjr8dC5iFyAI+h/5GzyAaYCjxljpgBNwAlj/ADGmMeNMVnGmKyYmBg3ylJKnUlSooM55OzR/+PTA2T9agV1Lb1fLWu3G3YW12FM97jZVVLP917axn835Pdrvd7EnaAvAhJdnicAJccvJCIZwBPAQmNMlcu6RcaYDc7nr+AIfqXUEDN2eAiFNc08s/Ywf1ixj6b2TnaV1PW4bF1LB7c9s4kr/rqGf6/rHuhbChyjxBsPVfd7zd7CnaDfBIwVkRQR8QOuA95wXUBEkoDlwE3GmH1H240xR4BCERnnbJoH7O6TypVSZ5SbZo1i4shQfv7GLvysjujZXVJ/wnKF1c0s/vtnrN5fSWpMMA+/v5dyl9MytzrvR7vpcI2el++mUwa9McYG3AW8j+OMmZeMMbtE5A4RucO52M+AKODvIpIjItkub/Ft4DkR2Q5kAr/uyw1QSp0ZIoP9eP4bM/ny2Qn8cclkhof4s7u0nq0FNcz/0yo2HqpmS0ENX3psLZUNbTz39Rk8cXMWbTY7l/91DRf/8VN2lziW9/OxUNfSwf5y98/i2VVSx41PrOdARfd1jDHsK2s4YYhooK3ZX8kLGwvo6LT3+XuLpzeuJ1lZWSY7O/vUCyqlzlhfe3ojR+payUgI46XsIvx8LNjthhFhATz1tWmkxYYA8Pb2Ut7aXsKa/ZWkx4ex7mAV109P4vmNBTy4KJ2bZo7q9TMqG9tYvb+CRZnx3PNiDv/LKSEuLICXvjmr61aHj686wK/f2cOzt83g3LHRA7LtPVnyj3WU1Lbw6Q8vwGrp6dDoyYnIZmNMVk+v6ZWxSimPmBAXSl55Ix/mlnNeWgwzUiKZnz6Ct78zpyvkAS7PiOOxr5zNTbNGse6g4/DfwsyRxIb6s+kU4/T3vrKde17cxkvZhby36whzxkbT1Gbjl286RpDXH6xi6Xt7Afh0X/lp1W+Mod3WN73v3NJ6Nh6q5uZZoz5XyJ+KBr1SyiMmjAzFZjdUN7Xzpanx/Oe2GfzthqmEBfr2uPzXZifjZ7VgtQgZCWFMT4li7YFKWjs6WbG7jFv/tYl/fHqAz/IqKa5t4cPdZXy0pxx/HwsPvLaT1g4791ycxuIp8azJq6C1o5Of/m8nSZFBTE4MZ+2BKkpqW1jw59XsPdJ9ps3Kxjbm/2kVK3PLutr+sz6f6b/+kKKa5q62Z9Ye5t5XtvHnD/fT2uH+6aP/XncYfx8LS7IST73w5+DTL++qlFKnMHFkGAA+FuH8ccNPufzw0ABuPTeF/Komgvx8uGF6Em9uK+GxTw7w3IZ8mts7+WhP9155akwwP15wFl//dzap0cFMSQynrqWDZ9bl88zaw+wvb+RXi9KpaWrnjx/u45EV+8gtreejPeWMG3Hsr4pfv5PLniMNvLipkHlnxQLwek4Jtc0dPPDaTv51yzTabHYeejsXX6vQ1N5Jbmk9j9449ZQ99LqWDl7bWsyizHjCg/xOdze6RYNeKeURoyKDCPazkpkU3msv/nj3LRjf9XjW6CjOHRPNn1fuRwTeuPNcYsP8yStr5FBVEwXVzVyZMZKJI0P55nmpZCaEIyLMSo3C38fCIx/uw8ciXDYpjkOVTfxhxT5e3lwEwI7i2q7PWXegiuVbigkL9GX1fsdfEE1tNrYU1JAaE8yn+yp4Y1sJI0IDaO+08+iNWRRUN/PgW7v5y8r93HNxWo/bYoxBRHhjWwmtHXZunJn0+XfmKejQjVLKIywW4ZFrM/nJ5cdPneW+H1zqOHP7ummJTEoIY3hIALPHRHPjjFHcv+As0uPDEBHuX3AWCybFARDga2XW6ChaO+zMGRtNZLAfGQlhBPtZAUiOCmJ7keP8fmMMv3k3l/jwQH53TQYtHZ18llfJx3srMAYeWZLJ6JhgnltfQHa+4/z+rFER3HZuCueMieL9XUe6ajXGkH24mnabnbV5lUx7aCXv7zrCK9mFjB8RwqT4sM+9H05Fe/RKKY+5ZOKIL7R+ZmI4H9wzt2t2THednxbDJ3srWJjpmM3F12rhwrNiOVLXwryzYvntu3uoaWpnS0EN24vq+N3VGZw/LoZh/j58mFtGbXMHsaH+ZCSEcdXkeP60ch+NbTbGDh9GRLBj+GVaciR/XrmfxjYbw/x9WLG7jNv/s5n0+FDyq5ppaLXx/Ze20dhm4yeXn4VI3x+EPUp79EqpM1pabAi+1tOLsqvPTuBH88ezYNKxXzSPLJnMc1+fSYazZ729uI4/fbifpMggFk+Nx9/HynlpMTy/sZB3dx7hwvGxiAiXZ4zAGNhdWs+0lMiu95uSFIExsL2oFoA3t5cSEuBDUU0LAb5W/nPbdGx2Oz4WYfGUk00f9sVpj14pNeSEBPjyrfNHd2vzcf6ymOgM+gff2k1eeSMPf3ly1y+S712SRnJ0EAE+Vq7JSgBgzPAQxo8IYc+RBqYnHwv6zIRwwHEl75TECFbmlrFoSjw/unQ8ncYQGezHn67NpKy+jahh/v26vRr0SinlIizQl5ToYPLKG7norFiunnqstz06Zhg/vHT8CetcOXkk+8r2duvRhwX5khoTzNaCWj6JLqe5vZPLJ8URFnTswPP89Lj+3RgnDXqllDrOzNRI2m12Hv5yhltj59+Yk8p5aTHEhwd2a5+SGMEne8upb+kgKtiPGS6/CAaSBr1SSh3n/y1Mp6PTTpCfexHp52MhvYezZjKTwnl1SxF1LTX84qqJXcNDA02DXimljuNrtZz2Ad6eXJkRx6GKJq6bnthtWoeBpkGvlFL9JDzIj59d+fmvE+grenqlUkp5OQ16pZTychr0Sinl5TTolVLKy2nQK6WUl9OgV0opL6dBr5RSXk6DXimlvJwYYzxdwwlEpALI/5yrRwOVfVhOX9G6Tt9grU3rOj1a1+n7PLWNMsbE9PTCoAz6L0JEso0xWZ6u43ha1+kbrLVpXadH6zp9fV2bDt0opZSX06BXSikv541B/7inC+iF1nX6BmttWtfp0bpOX5/W5nVj9Eoppbrzxh69UkopFxr0Sinl5bwm6EVkvojsFZE8EbnPg3UkisjHIpIrIrtE5G5n+y9EpFhEcpz/LvNQfYdFZIezhmxnW6SIrBCR/c7/RgxwTeNc9kuOiNSLyHc9sc9E5CkRKReRnS5tve4fEbnf+Z3bKyKXeqC234vIHhHZLiKviUi4sz1ZRFpc9t2yAa6r15/dQO2zXup60aWmwyKS42wfyP3VW0b03/fMGHPG/wOswAEgFfADtgETPFRLHDDV+TgE2AdMAH4B/GAQ7KvDQPRxbb8D7nM+vg9Y6uGf5RFglCf2GTAXmArsPNX+cf5ctwH+QIrzO2gd4NouAXycj5e61JbsupwH9lmPP7uB3Gc91XXc638AfuaB/dVbRvTb98xbevTTgTxjzEFjTDvwArDQE4UYY0qNMVucjxuAXCDeE7WchoXAM87HzwCLPFcK84ADxpjPe2X0F2KMWQVUH9fc2/5ZCLxgjGkzxhwC8nB8FwesNmPMB8YYm/PpeiChvz7/dOo6iQHbZyerS0QEWAI83x+ffTInyYh++555S9DHA4Uuz4sYBOEqIsnAFGCDs+ku55/YTw308IgLA3wgIptF5HZnW6wxphQcX0JguIdqA7iO7v/zDYZ91tv+GWzfu1uBd12ep4jIVhH5VETmeKCenn52g2WfzQHKjDH7XdoGfH8dlxH99j3zlqCXHto8et6oiAwDXgW+a4ypBx4DRgOZQCmOPxs94RxjzFRgAXCniMz1UB0nEBE/4CrgZWfTYNlnvRk03zsReQCwAc85m0qBJGPMFOB7wH9FJHQAS+rtZzdY9tn1dO9QDPj+6iEjel20h7bT2mfeEvRFQKLL8wSgxEO1ICK+OH6AzxljlgMYY8qMMZ3GGDvwT/rxT/yTMcaUOP9bDrzmrKNMROKctccB5Z6oDccvny3GmDJnjYNin9H7/hkU3zsR+SpwBXCjcQ7qOv/Mr3I+3oxjXDdtoGo6yc/O4/tMRHyALwEvHm0b6P3VU0bQj98zbwn6TcBYEUlx9gqvA97wRCHOsb8ngVxjzB9d2uNcFlsM7Dx+3QGoLVhEQo4+xnEgbyeOffVV52JfBV4f6NqcuvWyBsM+c+pt/7wBXCci/iKSAowFNg5kYSIyH/gRcJUxptmlPUZErM7Hqc7aDg5gXb397Dy+z4CLgD3GmKKjDQO5v3rLCPrzezYQR5kH6Ej2ZTiOXh8AHvBgHefi+LNqO5Dj/HcZ8B9gh7P9DSDOA7Wl4jh6vw3YdXQ/AVHASmC/87+RHqgtCKgCwlzaBnyf4fhFUwp04OhJ3Xay/QM84PzO7QUWeKC2PBzjt0e/a8ucy17t/BlvA7YAVw5wXb3+7AZqn/VUl7P9X8Adxy07kPurt4zot++ZToGglFJezluGbpRSSvVCg14ppbycBr1SSnk5DXqllPJyGvRKKeXlNOiVUsrLadArpZSX+/8hNOvuG3L/RgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(torch.tensor(lossi).view(-1, 1000).mean(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# put layers into eval mode (needed for batchnorm especially)\n",
    "for layer in model.layers:\n",
    "  layer.training = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train 1.7690284252166748\n",
      "val 1.9936515092849731\n"
     ]
    }
   ],
   "source": [
    "# evaluate the loss\n",
    "@torch.no_grad() # this decorator disables gradient tracking inside pytorch\n",
    "def split_loss(split):\n",
    "  x,y = {\n",
    "    'train': (Xtr, Ytr),\n",
    "    'val': (Xdev, Ydev),\n",
    "    'test': (Xte, Yte),\n",
    "  }[split]\n",
    "  logits = model(x)\n",
    "  loss = F.cross_entropy(logits, y)\n",
    "  print(split, loss.item())\n",
    "\n",
    "split_loss('train')\n",
    "split_loss('val')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### performance log\n",
    "\n",
    "- original (3 character context + 200 hidden neurons, 12K params): train 2.058, val 2.105\n",
    "- context: 3 -> 8 (22K params): train 1.918, val 2.027\n",
    "- flat -> hierarchical (22K params): train 1.941, val 2.029\n",
    "- fix bug in batchnorm: train 1.912, val 2.022\n",
    "- scale up the network: n_embd 24, n_hidden 128 (76K params): train 1.769, val 1.993\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arlij.\n",
      "chetta.\n",
      "heago.\n",
      "rocklei.\n",
      "hendrix.\n",
      "jamylie.\n",
      "broxin.\n",
      "denish.\n",
      "anslibt.\n",
      "marianah.\n",
      "astavia.\n",
      "annayve.\n",
      "aniah.\n",
      "jayce.\n",
      "nodiel.\n",
      "remita.\n",
      "niyelle.\n",
      "jaylene.\n",
      "aiyan.\n",
      "aubreana.\n"
     ]
    }
   ],
   "source": [
    "# sample from the model\n",
    "for _ in range(20):\n",
    "    \n",
    "    out = []\n",
    "    context = [0] * block_size # initialize with all ...\n",
    "    while True:\n",
    "      # forward pass the neural net\n",
    "      logits = model(torch.tensor([context]))\n",
    "      probs = F.softmax(logits, dim=1)\n",
    "      # sample from the distribution\n",
    "      ix = torch.multinomial(probs, num_samples=1).item()\n",
    "      # shift the context window and track the samples\n",
    "      context = context[1:] + [ix]\n",
    "      out.append(ix)\n",
    "      # if we sample the special '.' token, break\n",
    "      if ix == 0:\n",
    "        break\n",
    "    \n",
    "    print(''.join(itos[i] for i in out)) # decode and print the generated word"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Next time:\n",
    "Why convolutions? Brief preview/hint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........ --> d\n",
      ".......d --> i\n",
      "......di --> o\n",
      ".....dio --> n\n",
      "....dion --> d\n",
      "...diond --> r\n",
      "..diondr --> e\n",
      ".diondre --> .\n"
     ]
    }
   ],
   "source": [
    "for x,y in zip(Xtr[7:15], Ytr[7:15]):\n",
    "  print(''.join(itos[ix.item()] for ix in x), '-->', itos[y.item()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 27])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# forward a single example:\n",
    "logits = model(Xtr[[7]])\n",
    "logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 27])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# forward all of them\n",
    "logits = torch.zeros(8, 27)\n",
    "for i in range(8):\n",
    "  logits[i] = model(Xtr[[7+i]])\n",
    "logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convolution is a \"for loop\"\n",
    "# allows us to forward Linear layers efficiently over space"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}