\n", "\n", "Besides initialization, selecting a suitable optimization algorithm can be an important choice for deep neural networks.\n", "Before taking a closer look at them, we should define code for training the models.\n", "Most of the following code is copied from the previous tutorial, and only slightly altered to fit our needs."]}, {"cell_type": "code", "execution_count": 20, "id": "6ab60b47", "metadata": {"execution": {"iopub.execute_input": "2023-03-15T10:01:57.128300Z", "iopub.status.busy": "2023-03-15T10:01:57.127537Z", "iopub.status.idle": "2023-03-15T10:01:57.153229Z", "shell.execute_reply": "2023-03-15T10:01:57.152457Z"}, "lines_to_next_cell": 2, "papermill": {"duration": 0.090009, "end_time": "2023-03-15T10:01:57.154785", "exception": false, "start_time": "2023-03-15T10:01:57.064776", "status": "completed"}, "tags": []}, "outputs": [], "source": ["def _get_config_file(model_path, model_name):\n", " return os.path.join(model_path, model_name + \".config\")\n", "\n", "\n", "def _get_model_file(model_path, model_name):\n", " return os.path.join(model_path, model_name + \".tar\")\n", "\n", "\n", "def _get_result_file(model_path, model_name):\n", " return os.path.join(model_path, model_name + \"_results.json\")\n", "\n", "\n", "def load_model(model_path, model_name, net=None):\n", " config_file = _get_config_file(model_path, model_name)\n", " model_file = _get_model_file(model_path, model_name)\n", " assert os.path.isfile(\n", " config_file\n", " ), f'Could not find the config file \"{config_file}\". Are you sure this is the correct path and you have your model config stored here?'\n", " assert os.path.isfile(\n", " model_file\n", " ), f'Could not find the model file \"{model_file}\". Are you sure this is the correct path and you have your model stored here?'\n", " with open(config_file) as f:\n", " config_dict = json.load(f)\n", " if net is None:\n", " act_fn_name = config_dict[\"act_fn\"].pop(\"name\").lower()\n", " assert (\n", " act_fn_name in act_fn_by_name\n", " ), f'Unknown activation function \"{act_fn_name}\". Please add it to the \"act_fn_by_name\" dict.'\n", " act_fn = act_fn_by_name[act_fn_name]()\n", " net = BaseNetwork(act_fn=act_fn, **config_dict)\n", " net.load_state_dict(torch.load(model_file))\n", " return net\n", "\n", "\n", "def save_model(model, model_path, model_name):\n", " config_dict = model.config\n", " os.makedirs(model_path, exist_ok=True)\n", " config_file = _get_config_file(model_path, model_name)\n", " model_file = _get_model_file(model_path, model_name)\n", " with open(config_file, \"w\") as f:\n", " json.dump(config_dict, f)\n", " torch.save(model.state_dict(), model_file)\n", "\n", "\n", "def train_model(net, model_name, optim_func, max_epochs=50, batch_size=256, overwrite=False):\n", " \"\"\"Train a model on the training set of FashionMNIST.\n", "\n", " Args:\n", " net: Object of BaseNetwork\n", " model_name: (str) Name of the model, used for creating the checkpoint names\n", " max_epochs: Number of epochs we want to (maximally) train for\n", " patience: If the performance on the validation set has not improved for #patience epochs, we stop training early\n", " batch_size: Size of batches used in training\n", " overwrite: Determines how to handle the case when there already exists a checkpoint. If True, it will be overwritten. Otherwise, we skip training.\n", " \"\"\"\n", " file_exists = os.path.isfile(_get_model_file(CHECKPOINT_PATH, model_name))\n", " if file_exists and not overwrite:\n", " print(f'Model file of \"{model_name}\" already exists. Skipping training...')\n", " with open(_get_result_file(CHECKPOINT_PATH, model_name)) as f:\n", " results = json.load(f)\n", " else:\n", " if file_exists:\n", " print(\"Model file exists, but will be overwritten...\")\n", "\n", " # Defining optimizer, loss and data loader\n", " optimizer = optim_func(net.parameters())\n", " loss_module = nn.CrossEntropyLoss()\n", " train_loader_local = data.DataLoader(\n", " train_set, batch_size=batch_size, shuffle=True, drop_last=True, pin_memory=True\n", " )\n", "\n", " results = None\n", " val_scores = []\n", " train_losses, train_scores = [], []\n", " best_val_epoch = -1\n", " for epoch in range(max_epochs):\n", " train_acc, val_acc, epoch_losses = epoch_iteration(\n", " net, loss_module, optimizer, train_loader_local, val_loader, epoch\n", " )\n", " train_scores.append(train_acc)\n", " val_scores.append(val_acc)\n", " train_losses += epoch_losses\n", "\n", " if len(val_scores) == 1 or val_acc > val_scores[best_val_epoch]:\n", " print(\"\\t (New best performance, saving model...)\")\n", " save_model(net, CHECKPOINT_PATH, model_name)\n", " best_val_epoch = epoch\n", "\n", " if results is None:\n", " load_model(CHECKPOINT_PATH, model_name, net=net)\n", " test_acc = test_model(net, test_loader)\n", " results = {\n", " \"test_acc\": test_acc,\n", " \"val_scores\": val_scores,\n", " \"train_losses\": train_losses,\n", " \"train_scores\": train_scores,\n", " }\n", " with open(_get_result_file(CHECKPOINT_PATH, model_name), \"w\") as f:\n", " json.dump(results, f)\n", "\n", " # Plot a curve of the validation accuracy\n", " sns.set()\n", " plt.plot([i for i in range(1, len(results[\"train_scores\"]) + 1)], results[\"train_scores\"], label=\"Train\")\n", " plt.plot([i for i in range(1, len(results[\"val_scores\"]) + 1)], results[\"val_scores\"], label=\"Val\")\n", " plt.xlabel(\"Epochs\")\n", " plt.ylabel(\"Validation accuracy\")\n", " plt.ylim(min(results[\"val_scores\"]), max(results[\"train_scores\"]) * 1.01)\n", " plt.title(f\"Validation performance of {model_name}\")\n", " plt.legend()\n", " plt.show()\n", " plt.close()\n", "\n", " print((f\" Test accuracy: {results['test_acc']*100.0:4.2f}% \").center(50, \"=\") + \"\\n\")\n", " return results\n", "\n", "\n", "def epoch_iteration(net, loss_module, optimizer, train_loader_local, val_loader, epoch):\n", " ############\n", " # Training #\n", " ############\n", " net.train()\n", " true_preds, count = 0.0, 0\n", " epoch_losses = []\n", " t = tqdm(train_loader_local, leave=False)\n", " for imgs, labels in t:\n", " imgs, labels = imgs.to(device), labels.to(device)\n", " optimizer.zero_grad()\n", " preds = net(imgs)\n", " loss = loss_module(preds, labels)\n", " loss.backward()\n", " optimizer.step()\n", " # Record statistics during training\n", " true_preds += (preds.argmax(dim=-1) == labels).sum().item()\n", " count += labels.shape[0]\n", " t.set_description(f\"Epoch {epoch+1}: loss={loss.item():4.2f}\")\n", " epoch_losses.append(loss.item())\n", " train_acc = true_preds / count\n", "\n", " ##############\n", " # Validation #\n", " ##############\n", " val_acc = test_model(net, val_loader)\n", " print(\n", " f\"[Epoch {epoch+1:2i}] Training accuracy: {train_acc*100.0:05.2f}%, Validation accuracy: {val_acc*100.0:05.2f}%\"\n", " )\n", " return train_acc, val_acc, epoch_losses\n", "\n", "\n", "def test_model(net, data_loader):\n", " \"\"\"Test a model on a specified dataset.\n", "\n", " Args:\n", " net: Trained model of type BaseNetwork\n", " data_loader: DataLoader object of the dataset to test on (validation or test)\n", " \"\"\"\n", " net.eval()\n", " true_preds, count = 0.0, 0\n", " for imgs, labels in data_loader:\n", " imgs, labels = imgs.to(device), labels.to(device)\n", " with torch.no_grad():\n", " preds = net(imgs).argmax(dim=-1)\n", " true_preds += (preds == labels).sum().item()\n", " count += labels.shape[0]\n", " test_acc = true_preds / count\n", " return test_acc"]}, {"cell_type": "markdown", "id": "90c3ac7b", "metadata": {"lines_to_next_cell": 2, "papermill": {"duration": 0.054123, "end_time": "2023-03-15T10:01:57.265753", "exception": false, "start_time": "2023-03-15T10:01:57.211630", "status": "completed"}, "tags": []}, "source": ["First, we need to understand what an optimizer actually does.\n", "The optimizer is responsible to update the network's parameters given the gradients.\n", "Hence, we effectively implement a function $w^{t} = f(w^{t-1}, g^{t}, ...)$ with $w$ being the parameters, and $g^{t} = \\nabla_{w^{(t-1)}} \\mathcal{L}^{(t)}$ the gradients at time step $t$.\n", "A common, additional parameter to this function is the learning rate, here denoted by $\\eta$.\n", "Usually, the learning rate can be seen as the \"step size\" of the update.\n", "A higher learning rate means that we change the weights more in the direction of the gradients, a smaller means we take shorter steps.\n", "\n", "As most optimizers only differ in the implementation of $f$, we can define a template for an optimizer in PyTorch below.\n", "We take as input the parameters of a model and a learning rate.\n", "The function `zero_grad` sets the gradients of all parameters to zero, which we have to do before calling `loss.backward()`.\n", "Finally, the `step()` function tells the optimizer to update all weights based on their gradients.\n", "The template is setup below:"]}, {"cell_type": "code", "execution_count": 21, "id": "3fc98bf2", "metadata": {"execution": {"iopub.execute_input": "2023-03-15T10:01:57.377724Z", "iopub.status.busy": "2023-03-15T10:01:57.377347Z", "iopub.status.idle": "2023-03-15T10:01:57.386754Z", "shell.execute_reply": "2023-03-15T10:01:57.386006Z"}, "lines_to_next_cell": 2, "papermill": {"duration": 0.067977, "end_time": "2023-03-15T10:01:57.388413", "exception": false, "start_time": "2023-03-15T10:01:57.320436", "status": "completed"}, "tags": []}, "outputs": [], "source": ["class OptimizerTemplate:\n", " def __init__(self, params, lr):\n", " self.params = list(params)\n", " self.lr = lr\n", "\n", " def zero_grad(self):\n", " # Set gradients of all parameters to zero\n", " for p in self.params:\n", " if p.grad is not None:\n", " p.grad.detach_() # For second-order optimizers important\n", " p.grad.zero_()\n", "\n", " @torch.no_grad()\n", " def step(self):\n", " # Apply update step to all parameters\n", " for p in self.params:\n", " if p.grad is None: # We skip parameters without any gradients\n", " continue\n", " self.update_param(p)\n", "\n", " def update_param(self, p):\n", " # To be implemented in optimizer-specific classes\n", " raise NotImplementedError"]}, {"cell_type": "markdown", "id": "99114a19", "metadata": {"lines_to_next_cell": 2, "papermill": {"duration": 0.058249, "end_time": "2023-03-15T10:01:57.504361", "exception": false, "start_time": "2023-03-15T10:01:57.446112", "status": "completed"}, "tags": []}, "source": ["The first optimizer we are going to implement is the standard Stochastic Gradient Descent (SGD).\n", "SGD updates the parameters using the following equation:\n", "\n", "$$\n", "\\begin{split}\n", " w^{(t)} & = w^{(t-1)} - \\eta \\cdot g^{(t)}\n", "\\end{split}\n", "$$\n", "\n", "As simple as the equation is also our implementation of SGD:"]}, {"cell_type": "code", "execution_count": 22, "id": "b54db8b8", "metadata": {"execution": {"iopub.execute_input": "2023-03-15T10:01:57.635864Z", "iopub.status.busy": "2023-03-15T10:01:57.635242Z", "iopub.status.idle": "2023-03-15T10:01:57.640922Z", "shell.execute_reply": "2023-03-15T10:01:57.640223Z"}, "lines_to_next_cell": 2, "papermill": {"duration": 0.067585, "end_time": "2023-03-15T10:01:57.642142", "exception": false, "start_time": "2023-03-15T10:01:57.574557", "status": "completed"}, "tags": []}, "outputs": [], "source": ["class SGD(OptimizerTemplate):\n", " def __init__(self, params, lr):\n", " super().__init__(params, lr)\n", "\n", " def update_param(self, p):\n", " p_update = -self.lr * p.grad\n", " p.add_(p_update) # In-place update => saves memory and does not create computation graph"]}, {"cell_type": "markdown", "id": "22bd85ea", "metadata": {"lines_to_next_cell": 2, "papermill": {"duration": 0.053912, "end_time": "2023-03-15T10:01:57.753533", "exception": false, "start_time": "2023-03-15T10:01:57.699621", "status": "completed"}, "tags": []}, "source": ["In the lecture, we also have discussed the concept of momentum which replaces the gradient in the update by an exponential average of all past gradients including the current one:\n", "\n", "$$\n", "\\begin{split}\n", " m^{(t)} & = \\beta_1 m^{(t-1)} + (1 - \\beta_1)\\cdot g^{(t)}\\\\\n", " w^{(t)} & = w^{(t-1)} - \\eta \\cdot m^{(t)}\\\\\n", "\\end{split}\n", "$$\n", "\n", "Let's also implement it below:"]}, {"cell_type": "code", "execution_count": 23, "id": "0d9787c1", "metadata": {"execution": {"iopub.execute_input": "2023-03-15T10:01:57.863419Z", "iopub.status.busy": "2023-03-15T10:01:57.863052Z", "iopub.status.idle": "2023-03-15T10:01:57.871487Z", "shell.execute_reply": "2023-03-15T10:01:57.870554Z"}, "lines_to_next_cell": 2, "papermill": {"duration": 0.065488, "end_time": "2023-03-15T10:01:57.873355", "exception": false, "start_time": "2023-03-15T10:01:57.807867", "status": "completed"}, "tags": []}, "outputs": [], "source": ["class SGDMomentum(OptimizerTemplate):\n", " def __init__(self, params, lr, momentum=0.0):\n", " super().__init__(params, lr)\n", " self.momentum = momentum # Corresponds to beta_1 in the equation above\n", " self.param_momentum = {p: torch.zeros_like(p.data) for p in self.params} # Dict to store m_t\n", "\n", " def update_param(self, p):\n", " self.param_momentum[p] = (1 - self.momentum) * p.grad + self.momentum * self.param_momentum[p]\n", " p_update = -self.lr * self.param_momentum[p]\n", " p.add_(p_update)"]}, {"cell_type": "markdown", "id": "910287c4", "metadata": {"lines_to_next_cell": 2, "papermill": {"duration": 0.054066, "end_time": "2023-03-15T10:01:57.986812", "exception": false, "start_time": "2023-03-15T10:01:57.932746", "status": "completed"}, "tags": []}, "source": ["Finally, we arrive at Adam.\n", "Adam combines the idea of momentum with an adaptive learning rate, which is based on an exponential average of the squared gradients, i.e. the gradients norm.\n", "Furthermore, we add a bias correction for the momentum and adaptive learning rate for the first iterations:\n", "\n", "$$\n", "\\begin{split}\n", " m^{(t)} & = \\beta_1 m^{(t-1)} + (1 - \\beta_1)\\cdot g^{(t)}\\\\\n", " v^{(t)} & = \\beta_2 v^{(t-1)} + (1 - \\beta_2)\\cdot \\left(g^{(t)}\\right)^2\\\\\n", " \\hat{m}^{(t)} & = \\frac{m^{(t)}}{1-\\beta^{t}_1}, \\hat{v}^{(t)} = \\frac{v^{(t)}}{1-\\beta^{t}_2}\\\\\n", " w^{(t)} & = w^{(t-1)} - \\frac{\\eta}{\\sqrt{v^{(t)}} + \\epsilon}\\circ \\hat{m}^{(t)}\\\\\n", "\\end{split}\n", "$$\n", "\n", "Epsilon is a small constant used to improve numerical stability for very small gradient norms.\n", "Remember that the adaptive learning rate does not replace the learning\n", "rate hyperparameter $\\eta$, but rather acts as an extra factor and\n", "ensures that the gradients of various parameters have a similar norm."]}, {"cell_type": "code", "execution_count": 24, "id": "61615c78", "metadata": {"execution": {"iopub.execute_input": "2023-03-15T10:01:58.097304Z", "iopub.status.busy": "2023-03-15T10:01:58.096308Z", "iopub.status.idle": "2023-03-15T10:01:58.109347Z", "shell.execute_reply": "2023-03-15T10:01:58.108516Z"}, "papermill": {"duration": 0.070582, "end_time": "2023-03-15T10:01:58.111616", "exception": false, "start_time": "2023-03-15T10:01:58.041034", "status": "completed"}, "tags": []}, "outputs": [], "source": ["class Adam(OptimizerTemplate):\n", " def __init__(self, params, lr, beta1=0.9, beta2=0.999, eps=1e-8):\n", " super().__init__(params, lr)\n", " self.beta1 = beta1\n", " self.beta2 = beta2\n", " self.eps = eps\n", " self.param_step = {p: 0 for p in self.params} # Remembers \"t\" for each parameter for bias correction\n", " self.param_momentum = {p: torch.zeros_like(p.data) for p in self.params}\n", " self.param_2nd_momentum = {p: torch.zeros_like(p.data) for p in self.params}\n", "\n", " def update_param(self, p):\n", " self.param_step[p] += 1\n", "\n", " self.param_momentum[p] = (1 - self.beta1) * p.grad + self.beta1 * self.param_momentum[p]\n", " self.param_2nd_momentum[p] = (1 - self.beta2) * (p.grad) ** 2 + self.beta2 * self.param_2nd_momentum[p]\n", "\n", " bias_correction_1 = 1 - self.beta1 ** self.param_step[p]\n", " bias_correction_2 = 1 - self.beta2 ** self.param_step[p]\n", "\n", " p_2nd_mom = self.param_2nd_momentum[p] / bias_correction_2\n", " p_mom = self.param_momentum[p] / bias_correction_1\n", " p_lr = self.lr / (torch.sqrt(p_2nd_mom) + self.eps)\n", " p_update = -p_lr * p_mom\n", "\n", " p.add_(p_update)"]}, {"cell_type": "markdown", "id": "8c8d597a", "metadata": {"papermill": {"duration": 0.05471, "end_time": "2023-03-15T10:01:58.224457", "exception": false, "start_time": "2023-03-15T10:01:58.169747", "status": "completed"}, "tags": []}, "source": ["### Comparing optimizers on model training\n", "\n", "After we have implemented three optimizers (SGD, SGD with momentum, and Adam), we can start to analyze and compare them.\n", "First, we test them on how well they can optimize a neural network on the FashionMNIST dataset.\n", "We use again our linear network, this time with a ReLU activation and the kaiming initialization, which we have found before to work well for ReLU-based networks.\n", "Note that the model is over-parameterized for this task, and we can achieve similar performance with a much smaller network (for example `100,100,100`).\n", "However, our main interest is in how well the optimizer can train *deep*\n", "neural networks, hence the over-parameterization."]}, {"cell_type": "code", "execution_count": 25, "id": "d7d987e5", "metadata": {"execution": {"iopub.execute_input": "2023-03-15T10:01:58.334703Z", "iopub.status.busy": "2023-03-15T10:01:58.334333Z", "iopub.status.idle": "2023-03-15T10:01:58.350587Z", "shell.execute_reply": "2023-03-15T10:01:58.349864Z"}, "papermill": {"duration": 0.074388, "end_time": "2023-03-15T10:01:58.352890", "exception": false, "start_time": "2023-03-15T10:01:58.278502", "status": "completed"}, "tags": []}, "outputs": [], "source": ["base_model = BaseNetwork(act_fn=nn.ReLU(), hidden_sizes=[512, 256, 256, 128])\n", "kaiming_init(base_model)"]}, {"cell_type": "markdown", "id": "4c2386f2", "metadata": {"papermill": {"duration": 0.054197, "end_time": "2023-03-15T10:01:58.465919", "exception": false, "start_time": "2023-03-15T10:01:58.411722", "status": "completed"}, "tags": []}, "source": ["For a fair comparison, we train the exact same model with the same seed with the three optimizers below.\n", "Feel free to change the hyperparameters if you want (however, you have to train your own model then)."]}, {"cell_type": "code", "execution_count": 26, "id": "a2f97064", "metadata": {"execution": {"iopub.execute_input": "2023-03-15T10:01:58.575678Z", "iopub.status.busy": "2023-03-15T10:01:58.575208Z", "iopub.status.idle": "2023-03-15T10:01:59.072902Z", "shell.execute_reply": "2023-03-15T10:01:59.072164Z"}, "papermill": {"duration": 0.555248, "end_time": "2023-03-15T10:01:59.075098", "exception": false, "start_time": "2023-03-15T10:01:58.519850", "status": "completed"}, "tags": []}, "outputs": [{"name": "stdout", "output_type": "stream", "text": ["Model file of \"FashionMNIST_SGD\" already exists. Skipping training...\n"]}, {"data": {"application/pdf": "JVBERi0xLjQKJazcIKu6CjEgMCBvYmoKPDwgL1R5cGUgL0NhdGFsb2cgL1BhZ2VzIDIgMCBSID4+CmVuZG9iago4IDAgb2JqCjw8IC9Gb250IDMgMCBSIC9YT2JqZWN0IDcgMCBSIC9FeHRHU3RhdGUgNCAwIFIgL1BhdHRlcm4gNSAwIFIKL1NoYWRpbmcgNiAwIFIgL1Byb2NTZXQgWyAvUERGIC9UZXh0IC9JbWFnZUIgL0ltYWdlQyAvSW1hZ2VJIF0gPj4KZW5kb2JqCjExIDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL1Jlc291cmNlcyA4IDAgUgovTWVkaWFCb3ggWyAwIDAgNDIxLjE0NSAzMzEuNDAyNjI1IF0gL0NvbnRlbnRzIDkgMCBSIC9Bbm5vdHMgMTAgMCBSID4+CmVuZG9iago5IDAgb2JqCjw8IC9MZW5ndGggMTIgMCBSIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nL1ZXW9bNxJ9v7/iPsYPpeeD5JCPST+MLbALtDG6D4vFwlCdJoGdNM7uFvvv9wwl+ZKyq6qRbBuCrSNqyDOcOZzhPf/m+r/vVtc/Xryav349nS/vVp8nnt/j9ctM83u8fpt5vsDrl4nw7naKwoFjwv839/+rcogkWRJAGt++naY30/lLGPg8U6hsORqlUh68iZW4ZrIy3/ncF8OAad/oaUo5FMwWJdScfF6skxWjeuzmHlOqgczW4Oa7A9YW/Wl+YFaTBZZZcg6Mv3fX89/nD/P5S3F2PH+P13u81j5bvDrBqzkGU5UkwyoXdJh/ej39MH/aGqbACfuxtd3eXmzQ6RP2iuavCB9lDuLG2GLRWVJgcnOr2+nV5XT+Hc/M8+Wbto+XP0//mF/Q2fzP+fL76dvL6Yc24VGEd8OIctAEK3Vg3MEnoMwkoTRrapQP4ZyelHO0UK0Q2ch5gU/BOdLaWq71oG3mp93nUkMm0jLmXwefgnORtbWYOcWDSD/pRgskTkTAaSDdwScgLRI31kQhdAeQlifdaTEkW0wcR8736Cko59SMaS2Z+CDKT7rPyjHEbJDUgXMHn4C0km2sRV/4AaT1SfdZYYxKLXXM6A4+BelYN9aYYj2I9NPudLVgjGzTkfQCn4J0pbU1/z1Iu+Ow032moJYpENcarP+6jF//9tePq7efn9JvG1uZgnCx2FVe99CX618z7WEHxyVJavuKGQjHMxCtHJRS1J7pgh1JtfjqkkXUM/DbXq75GbgiUoNWYSkd2Q48ki1zQYRIjETJ6n665TnoJg3RShHu6S7gsXSheFIrqxZGfb6Pbn1Sfd/SLTGknBL37VIHHku3UED1V8SSlf2JW+UZ6AqnkJMI1Y5uBx5JVxhNr6GTw/5W3k/3OXRKYkZrWSppT3cBj6WLtaYEGUhU694eE5Seg66Z/00193QX8Fi6piHHVAlHbt0vVfU5pEqphAol7mN5wY4kq+S3EpL9Cof2CRUH2ilP3MpXbo/R5TeJN4gA76lQfjqbzTlJ5Ywfm19c3bz7+erf7z5+mK9Wq//cXa3+dyKHhjT/NqEoqYX8AEsGF8T1lZJowpu8vVSK84/9/Y07frKISWux7ARvp4IzH6uGf3Boq6E59SBD02sEFgXM0bEiANEi3Ey1oIRh71wZHWKtOOD99OzuQjhqIM41eWYy2otciGl9/KTIVL22YMEgignWM6o/rJN9Urepggopz2gh0ErEotxwtOBJk3r/jCEcS27ml+sIRkGK+SP5Khl+VRK/POKKY8CPvWYnYzmCtaEHRWhQO8EcR+eH8t3rHkHsEQ632nBMBtWvEFUBk2oatc27XAnAccEqErONR41dMW3GcCRsRhB7DAu8I7GqApeC9BNFXw0cNhNUDNEoGgNOrNzMiIdvLYLDRtoVCTSq4UtTLhHez3Bcsw/3gKMoaMUYVBOIOg73FOwbe4GOM8xSju4egXsQHmABHO6MVKzZgXsSFpv9XgJLzhSj7+7SGCOPIJZE4l4QeAcbnZF8jqviq204vBNrpOpdMNSGSs5+CKMUDCgYUoYbMhpFhU3fdYV74HuKHvclRNJc3E7Xm/oKDMWVtPFwj1Vud0bmTY0mcvt+W8tgnkHLUKHg9GrBhmMM2YHAw/oRVQrqrfNRuMdy8mlRK5DCye6drjuUklDXWaRmBt6JWKdxw7kgktoyfW1asHnAUQdSTr5Z2NZAGU5x80hqxOB6OLyjxaxaw009hxxf+jOp8FpBwHdqf5xSlLz5caVI/f2zQj4wF6L8MaVAliZDhzfnBAchlMogGP2V9CgY7EFYR8GAkhqGt9jpBQMpATny02gUDEEIwk95RzBg3VLym+ZRMBRmYta4KxjIRIMmy65gQKfwObXI7wVDOJhBV+MoGKx+t1G5tYGdYPhUCl2wPAqG4xFh1AqyTjDQFsFpRtUGwXDzVZHFcRQM1uznDsdRL9jz37LFNOqFs2p5nke9YNcFKrLO/0UvkDvwJpI1j3rh0xYsJo5ywZApTAQDo1y4ecNi1ubv5YIFBpEBbWs7uWhnBGJwrTqLXLDiqElkLfA7uUA0eYrhDBrlYgidTi44ghUZN/Xq5MJjBFlItiMXkOlQEtJuRy58XrBprBa1cOtUcJbFUS3clxVzNud0asGoVRlHR9xRC4b2IqIhJKNcsFekFd6vo1w4Hp14GuXCnV/h4/KwsRPXgYD83H2Q01Vg/SXRY4+QYPHhI6jbxx9BYeyBT7CWkZ2B37dKIHJo8QfJQQXmltETbYz8Ttn2R3Xbr9d3bz7enXkRWMWoqP/ML26vPqyu549v5u/OptqeHK4/wUdXn9/ii3/9219eX/7r9cU3KPimbcF3/lL/7LPD98DKn3uC2JIytRO21gwt8EcY4odE83yHu6paSM5cetyzSreji4snmA+okQuCj11NHa4ENd5u2oC7IESfpptRCQK8Ht2tr0NXWy4ddjNlP5Pc3T3s5cc40T1yvySYW2Pd8m/usYXoMkHnk4deXfmz1VfTYwfu/CWleXu0ihPSZ0HNmnNpCeEFzgDhpC0PoN0M2ZcdtmaCehnSs/fK9vKsaRcEFyP9d35xd/XuQ9/ENN4bzl9WZGw5W8G6NiqwJb1gC+se+wLaKADzA0ngxzo5g9Cuf10Rhs5t+j9wd7v8CmVuZHN0cmVhbQplbmRvYmoKMTIgMCBvYmoKMTk5NgplbmRvYmoKMTAgMCBvYmoKWyBdCmVuZG9iagoxNyAwIG9iago8PCAvTGVuZ3RoIDE2NCAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeJw9kMERQyEIRO9WsSWAgEA9yWRy+L//a0CTXGQdYPepO4GQUYczw2fiyYPTsTRwbxWMawivI/QITQKTwMTBmngMCwGnYZFjLt9VllWnla6ajZ7XvWNB1WmXNQ1t2oHyrY8/wjXeo/Aa7B5CB7EodG5lWguZWDxrnDvMo8znfk7bdz0YrabUrDdy2dc9OsvUUF5a+4TOaLT9J9cvuzFeH4UUOQgKZW5kc3RyZWFtCmVuZG9iagoxOCAwIG9iago8PCAvTGVuZ3RoIDgxIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nE3Nuw3AIAwE0J4pPALg/z5RlCLZv40NEaGxn3QnnWCHCm5xWAy0Oxyt+NRTmH3oHhKSUHPdRFgzJdqEpF/6yzDDmFjItq83V65yvhbcHIsKZW5kc3RyZWFtCmVuZG9iagoxOSAwIG9iago8PCAvTGVuZ3RoIDc2IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nDM1N1UwULC0ABKmhuYK5kaWCimGXEA+iJXLBRPLAbPMTMyALENLZJaJsSGQZWJhhsQyNrGAyiJYBkAabE0OzPQcrgyuNAA1FxkFCmVuZHN0cmVhbQplbmRvYmoKMjAgMCBvYmoKPDwgL0xlbmd0aCAyNDcgL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCnicTVFJbsQwDLv7FfzAAJasxXlPikEP7f+vJR0U7cEQI0tc4u7ERBZetlDXQofjw0ZeCZuB74PWnPgaseI/2kaklT9UWyATMVEkdFE3GvdIN7wK0X6kgleq91jzEXcrzVs6drG/98G05pEqq0I85Ngc2Uha10TR8T203nNDdMoggT43IQdEaY5ehaS/9sN1bTS7tTazJ6qDR6aE8kmzGprTKWbIbKjHbSpWMgo3qoyK+1RGWg/yNs4ygJPjhDJaT3asJqL81CeXkBcTccIuOzsWYhMLG4e0H5U+sfx86834m2mtpZBxQSI0xaXfZ7zH53j/AJVPXCYKZW5kc3RyZWFtCmVuZG9iagoyMSAwIG9iago8PCAvTGVuZ3RoIDUxIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nDOyNFUwULC0ABKGluYK5kaWCimGXEA+iJXLBRPLAbMMgDRYaQ5MRQ5XBlcaAL+MDVYKZW5kc3RyZWFtCmVuZG9iagoyMiAwIG9iago8PCAvTGVuZ3RoIDkwIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nD2Oyw3AMAhD70zBCOFTAvtUVQ/J/teGfHrBD1vIuAkWDB+j2oWVA2+CsSd1YF1eAxVCFhlk5Ns7F4tKZha/miapE9Ikcd5EoTtNSp0PtNPb4IXnA/XpHewKZW5kc3RyZWFtCmVuZG9iagoyMyAwIG9iago8PCAvTGVuZ3RoIDc3IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nDWNwQ3AMAgD/0zBCDiFUPapqj7S/b8tRHzsMwjserJwpEwT9hF8gf6c9NI4ULTITBlo2rO+2CS5g5cjlCea0qti9edFD90fyZ4YDAplbmRzdHJlYW0KZW5kb2JqCjI0IDAgb2JqCjw8IC9MZW5ndGggMzQxIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nDVSO9KbQQjrv1PoAp5Z3st5nMmk+HP/NgI7FSywQgLSAgeZeIkhqlGu+CVPMF4n8He9PI2fx7uQWvBUpB+4Nm3j/VizJgqWRiyF2ce+HyXkeGr8GwI9F2nCjExGDiQDcb/W5896kymH34A0bU4fJUkPogW7W8OOLwsySHpSw5Kd/LCuBVYXoQlzY00kI6dWpub52DNcxhNjJKiaBSTpE/epghFpxmPnrCUPMhxP9eLFr7fxWuYx9bKqQMY2wRxsJzPhFEUE4heUJDdxF00dxdHMWHO70FBS5L67h5OTXveXk6jAKyGcxVrCMUNPWeZkp0EJVK2cADOs174wTtNGCXdqur0r9vXzzCSM2xx2VkqmwTkO7mWTOYJkrzsmbMLjEPPePYKRmDe/iy2CK5c512T6sR9FG+mD4vqcqymzFSX8Q5U8seIa/5/f+/nz/P4HjCh+IwplbmRzdHJlYW0KZW5kb2JqCjI1IDAgb2JqCjw8IC9MZW5ndGggNjYgL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCnicMzM0VDBQ0DUCEmaGJgrmRpYKKYZcQD6IlcsFE8sBs8xMzIAsY1NTJJYBkDYyNYPTEBmgAXAGRH8GVxoAUmsUwAplbmRzdHJlYW0KZW5kb2JqCjI2IDAgb2JqCjw8IC9MZW5ndGggNzIgL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCnicM7MwUTBQsABiM3MzBXMjS4UUQy4jCzOgQC6XBVggh8vQ0BDKMjYxUjA0NAWyTM2NoWIwjUBZS5BBOVD9OVwZXGkAdDISoQplbmRzdHJlYW0KZW5kb2JqCjI3IDAgb2JqCjw8IC9MZW5ndGggMzA3IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nD2SS24DMQxD9z6FLhDA+tme86Qoupjef9snJemKHNkWRWqWukxZUx6QNJOEf+nwcLGd8jtsz2Zm4Fqil4nllOfQFWLuonzZzEZdWSfF6oRmOrfoUTkXBzZNqp+rLKXdLngO1yaeW/YRP7zQoB7UNS4JN3RXo2UpNGOq+3/Se/yMMuBqTF1sUqt7HzxeRFXo6AdHiSJjlxfn40EJ6UrCaFqIlXdFA0Hu8rTKewnu295qyLIHqZjOOylmsOt0Ui5uF4chHsjyqPDlo9hrQs/4sCsl9EjYhjNyJ+5oxubUyOKQ/t6NBEuPrmgh8+CvbtYuYLxTOkViZE5yrGmLVU73UBTTucO9DBD1bEVDKXOR1epfw84La5ZsFnhK+gUeo90mSw5W2duoTu+tPNnQ9x9a13QfCmVuZHN0cmVhbQplbmRvYmoKMjggMCBvYmoKPDwgL0xlbmd0aCAyMzIgL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCnicNVFJbsQwDLv7FfzAANbuvCfFoIf2/9dSyhQIQCW2uCViYyMCLzH4OYjc+JI1oyZ+Z3JX/CxPhUfCreBJFIGX4V52gssbxmU/DjMfvJdWzqTGkwzIRTY9PBEy2CUQOjC7BnXYZtqJviHhsyNSzUaW09cS9NIqBMpTtt/pghJtq/pz+6wLbfvaE052e+pJ5ROI55aswGXjFZPFWAY9UblLMX2Q6myhJ6G8KJ+DbD5qiESXKGfgicHBKNAO7LntZ+JVIWhd3adtY6hGSsfTvw1NTZII+UQJZ7Y07hb+f8+9vtf7D04hVBEKZW5kc3RyZWFtCmVuZG9iagoyOSAwIG9iago8PCAvTGVuZ3RoIDIzMSAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeJw1TzmSBCEMy3mFPjBVGNtAv6entjbY+X+6kplOkPAhydMTHZl4mSMjsGbH21pkIGbgU0zFv/a0DxOq9+AeIpSLC2GGkXDWrONuno4X/3aVz1gH7zb4illeENjCTNZXFmcu2wVjaZzEOclujF0TsY11radTWEcwoQyEdLbDlCBzVKT0yY4y5ug4kSeei+/22yx2OX4O6ws2jSEV5/gqeoI2g6Lsee8CGnJB/13d+B5Fu+glIBsJFtZRYu6c5YRfvXZ0HrUoEnNCmkEuEyHN6SqmEJpQrLOjoFJRcKk+p+isn3/lX1wtCmVuZHN0cmVhbQplbmRvYmoKMzAgMCBvYmoKPDwgL0xlbmd0aCAyNDkgL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCnicPVA7jkQhDOs5hS/wJPIjcB5Gqy1m79+uA5opUEx+tjMk0BGBRwwxlK/jJa2groG/i0LxbuLrg8Igq0NSIM56D4h07KY2kRM6HZwzP2E3Y47ARTEGnOl0pj0HJjn7wgqEcxtl7FZIJ4mqIo7qM44pnip7n3gWLO3INlsnkj3kIOFSUonJpZ+Uyj9typQKOmbRBCwSueBkE004y7tJUowZlDLqHqZ2In2sPMijOuhkTc6sI5nZ00/bmfgccLdf2mROlcd0Hsz4nLTOgzkVuvfjiTYHTY3a6Oz3E2kqL1K7HVqdfnUSld0Y5xgSl2d/Gd9k//kH/odaIgplbmRzdHJlYW0KZW5kb2JqCjMxIDAgb2JqCjw8IC9MZW5ndGggMzk1IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nD1SS27FQAjb5xRcoNLwm895UlXdvPtva0NSqSq8iTHGMH3KkLnlS10ScYXJt16uWzymfC5bWpl5iLuLjSU+ttyX7iG2XXQusTgdR/ILMp0qRKjNqtGh+EKWhQeQTvChC8J9Of7jL4DB17ANuOE9MkGwJOYpQsZuURmaEkERYeeRFaikUJ9Zwt9R7uv3MgVqb4ylC2Mc9Am0BUJtSMQC6kAAROyUVK2QjmckE78V3WdiHGDn0bIBrhlURJZ77MeIqc6ojLxExD5PTfoolkwtVsZuUxlf/JSM1Hx0BSqpNPKU8tBVs9ALWIl5EvY5/Ej459ZsIYY6btbyieUfM8UyEs5gSzlgoZfjR+DbWXURrh25uM50gR+V1nBMtOt+yPVP/nTbWs11vHIIokDlTUHwuw6uRrHExDI+nY0peqIssBqavEYzwWEQEdb3w8gDGv1yvBA0p2sitFgim7ViRI2KbHM9vQTWTO/FOdbDE8Js753WobIzMyohgtq6hmrrQHazvvNwtp8/M+iibQplbmRzdHJlYW0KZW5kb2JqCjMyIDAgb2JqCjw8IC9MZW5ndGggMTM2IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nE2PQQ4DMQgD73mFn0AgQHjPVlUP2/9fS9h20wseyYBsUQaBJYd4hxvh0dsP30U2FWfjnF9SKWIhmE9wnzBTHI0pd/Jjj4BxlGosp2h4XkvOTcMXLXcTLaWtl5MZb7jul/dHlW2RDUXPLQtC12yS+TKBB3wYmEd142mlx932bK/2/ADObDRJCmVuZHN0cmVhbQplbmRvYmoKMzMgMCBvYmoKPDwgL0xlbmd0aCAyNDkgL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCnicTVFJigMwDLvnFfpAIV6TvKdDmUPn/9fKDoU5BAmvkpOWmFgLDzGEHyw9+JEhczf9G36i2btZepLJ2f+Y5yJTUfhSqC5iQl2IG8+hEfA9oWsSWbG98Tkso5lzvgcfhbgEM6EBY31JMrmo5pUhE04MdRwOWqTCuGtiw+Ja0TyN3G77RmZlJoQNj2RC3BiAiCDrArIYLJQ2NhMyWc4D7Q3JDVpg16kbUYuCK5TWCXSiVsSqzOCz5tZ2N0Mt8uCoffH6aFaXYIXRS/VYeF+FPpipmXbukkJ64U07IsweCqQyOy0rtXvE6m6B+j/LUvD9yff4Ha8PzfxcnAplbmRzdHJlYW0KZW5kb2JqCjM0IDAgb2JqCjw8IC9MZW5ndGggOTQgL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCnicRY3BEcAgCAT/VEEJCgraTyaTh/b/jRAyfGDnDu6EBQu2eUYfBZUmXhVYB0pj3FCPQL3hci3J3AUPcCd/2tBUnJbTd2mRSVUp3KQSef8OZyaQqHnRY533C2P7IzwKZW5kc3RyZWFtCmVuZG9iagozNSAwIG9iago8PCAvTGVuZ3RoIDE2NCAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeJxFkMdxBTEMQ++qAiUwgAr1rMfzD+v+r4b000F6GEIMYk/CsFxXcWF0w4+3LTMNf0cZ7sb6MmO81VggJ+gDDJGJq9Gk+nbFGar05NVirqOiXC86IhLMkuOrQCN8OrLHk7a2M/10Xh/sIe8T/yoq525hAS6q7kD5Uh/x1I/ZUeqaoY8qK2seatpXhF0RSts+LqcyTt29A1rhvZWrPdrvPx52OvIKZW5kc3RyZWFtCmVuZG9iagozNiAwIG9iago8PCAvTGVuZ3RoIDcyIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nDMyt1AwULA0ARKGFiYK5mYGCimGXEC+qYm5Qi4XSAzEygGzDIC0JZyCiGeAmCBtEMUgFkSxmYkZRB2cAZHL4EoDACXbFskKZW5kc3RyZWFtCmVuZG9iagozNyAwIG9iago8PCAvTGVuZ3RoIDQ3IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nDMyt1AwULA0ARKGFiYK5mYGCimGXJYQVi4XTCwHzALRlnAKIp7BlQYAuWcNJwplbmRzdHJlYW0KZW5kb2JqCjM4IDAgb2JqCjw8IC9MZW5ndGggMjU4IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nEWRS3IEIAhE956CI4D85DyTSmUxuf82Dc5kNnaXqP2ESiOmEiznFHkwfcnyzWS26Xc5VjsbBRRFKJjJVeixAqs7U8SZa4lq62Nl5LjTOwbFG85dOalkcaOMdVR1KnBMz5X1Ud35dlmUfUcOZQrYrHMcbODKbcMYJ0abre4O94kgTydTR8XtINnwByeNfZWrK3CdbPbRSzAOBP1CE5jki0DrDIHGzVP05BLs4+N254Fgb3kRSNkQyJEhGB2Cdp1c/+LW+b3/cYY7z7UZrhzv4neY1nbHX2KSFXMBi9wpqOdrLlrXGTrekzPH5Kb7hs65YJe7g0zv+T/Wz/r+Ax4pZvoKZW5kc3RyZWFtCmVuZG9iagozOSAwIG9iago8PCAvTGVuZ3RoIDE2MyAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeJxFkDsSAyEMQ3tOoSP4IwM+z2YyKTb3b2PYbFLA01ggg7sTgtTagonogoe2Jd0F760EZ2P86TZuNRLkBHWAVqTjaJRSfbnFaZV08Wg2cysLrRMdZg56lKMZoBA6Fd7touRypu7O+UNw9V/1v2LdOZuJgcnKHQjN6lPc+TY7orq6yf6kx9ys134r7FVhaVlLywm3nbtmQAncUznaqz0/Hwo69gplbmRzdHJlYW0KZW5kb2JqCjQwIDAgb2JqCjw8IC9MZW5ndGggMzIyIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nDVRu23FMAzsNQUXMCB+Jc3jIEiRt3+bO9qpSNO8H1VeMqVcLnXJKllh8qVDdYqmfJ5mpvwO9ZDjmB7ZIbpT1pZ7GBaWiXlKHbGaLPdwCza+AJoScwvx9wjwK4BRwESgbvH3D7pZEkAaFPwU6JqrllhiAg2Lha3ZFeJW3SlYuKv4diS5BwlyMVnoUw5Fiim3wHwZLNmRWpzrclkK/259AhphhTjss4tE4HnAA0wk/mSAbM8+W+zq6kU2doY46dCAi4CbzSQBQVM4qz64Yftqu+bnmSgnODnWr6Ixvg1O5ktS3le5x8+gQd74Mzxnd45QDppQCPTdAiCH3cBGhD61z8AuA7ZJu3djSvmcZCm+BDYK9qhTHcrwYuzMVm/Y/MfoymZRbJCV9dHpDsrcoBNiHm9koVuytvs3D7N9/wFfGXtkCmVuZHN0cmVhbQplbmRvYmoKNDEgMCBvYmoKPDwgL0xlbmd0aCAyMTggL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCnicPVC5jQQxDMtdhRpYwHrtqWcWi0um//RI+fYi0RZFUio1mZIpL3WUJVlT3jp8lsQOeYblbmQ2JSpFL5OwJffQCvF9ieYU993VlrNDNJdoOX4LMyqqGx3TSzaacCoTuqDcwzP6DW10A1aHHrFbINCkYNe2IHLHDxgMwZkTiyIMSk0G/65yj59eixs+w/FDFJGSDuY1/1j98nMNr1OPJ5Fub77iXpypDgMRHJKavCNdWLEuEhFpNUFNz8BaLYC7t17+G7QjugxA9onEcZpSjqG/a3Clzy/lJ1PYCmVuZHN0cmVhbQplbmRvYmoKNDIgMCBvYmoKPDwgL0xlbmd0aCA4MyAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeJxFjLsNwDAIRHumYAR+JvY+UZTC3r8NECVuuCfdPVwdCZkpbjPDQwaeDCyGXXGB9JYwC1xHUI6d7KNh1b7qBI31plLz7w+Unuys4obrAQJCGmYKZW5kc3RyZWFtCmVuZG9iago0MyAwIG9iago8PCAvTGVuZ3RoIDIzOSAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeJxNUMltBDEM+7sKNTDA6By7HgeLPLL9f0PKCZKXaEviofKUW5bKZfcjOW/JuuVDh06VafJu0M2vsf6jDAJ2/1BUEK0lsUrMXNJusTRJL9nDOI2Xa7WO56l7hFmjePDj2NMpgek9MsFms705MKs9zg6QTrjGr+rTO5UkA4m6kPNCpQrrHtQloo8r25hSnU4t5RiXn+h7fI4APcXejdzRx8sXjEa1LajRapU4DzATU9GVcauRgZQTBkNnR1c0C6XIynpCNcKNOaGZvcNwYAPLs4Skpa1SvA9lAegCXdo64zRKgo4Awt8ojPX6Bqr8XjcKZW5kc3RyZWFtCmVuZG9iago0NCAwIG9iago8PCAvTGVuZ3RoIDUxIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nDM2tFAwUDA0MAeSRoZAlpGJQoohF0gAxMzlggnmgFkGQBqiOAeuJocrgysNAOG0DZgKZW5kc3RyZWFtCmVuZG9iago0NSAwIG9iago8PCAvTGVuZ3RoIDE2MCAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeJxFkDkSAzEIBHO9gidIXIL3rMu1wfr/qQfWR6LpAjQcuhZNynoUaD7psUahutBr6CxKkkTBFpIdUKdjiDsoSExIY5JIth6DI5pYs12YmVQqs1LhtGnFwr/ZWtXIRI1wjfyJ6QZU/E/qXJTwTYOvkjH6GFS8O4OMSfheRdxaMe3+RDCxGfYJb0UmBYSJsanZvs9ghsz3Ctc4x/MNTII36wplbmRzdHJlYW0KZW5kb2JqCjQ2IDAgb2JqCjw8IC9MZW5ndGggMzM0IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nC1SS3LFIAzbcwpdoDP4B+Q86XS6eL3/tpKTRUYOYPQx5YaJSnxZILej1sS3jcxAheGvq8yFz0jbyDqIy5CLuJIthXtELOQxxDzEgu+r8R4e+azMybMHxi/Zdw8r9tSEZSHjxRnaYRXHYRXkWLB1Iap7eFOkw6kk2OOL/z7Fcy0ELXxG0IBf5J+vjuD5khZp95ht0656sEw7qqSwHGxPc14mX1pnuToezwfJ9q7YEVK7AhSFuTPOc+Eo01ZGtBZ2NkhqXGxvjv1YStCFblxGiiOQn6kiPKCkycwmCuKPnB5yKgNh6pqudHIbVXGnnsw1m4u3M0lm675IsZnCeV04s/4MU2a1eSfPcqLUqQjvsWdL0NA5rp69lllodJsTvKSEz8ZOT06+VzPrITkVCaliWlfBaRSZYgnbEl9TUVOaehn++/Lu8Tt+/gEsc3xzCmVuZHN0cmVhbQplbmRvYmoKNDcgMCBvYmoKPDwgL0xlbmd0aCAzMjAgL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCnicNVJLbgUxCNvPKbhApfBPzvOqqou++29rE70VTDBg4ykvWdJLvtQl26XD5Fsf9yWxQt6P7ZrMUsX3FrMUzy2vR88Rty0KBFETPViZLxUi1M/06DqocEqfgVcItxQbvINJAINq+AcepTMgUOdAxrtiMlIDgiTYc2lxCIlyJol/pLye3yetpKH0PVmZy9+TS6XQHU1O6AHFysVJoF1J+aCZmEpEkpfrfbFC9IbAkjw+RzHJgOw2iW2iBSbnHqUlzMQUOrDHArxmmtVV6GDCHocpjFcLs6gebPJbE5WkHa3jGdkw3sswU2Kh4bAF1OZiZYLu5eM1r8KI7VGTXcNw7pbNdwjRaP4bFsrgYxWSgEensRINaTjAiMCeXjjFXvMTOQ7AiGOdmiwMY2gmp3qOicDQnrOlYcbHHlr18w9U6XyHCmVuZHN0cmVhbQplbmRvYmoKNDggMCBvYmoKPDwgL0xlbmd0aCAxOCAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeJwzNrRQMIDDFEOuNAAd5gNSCmVuZHN0cmVhbQplbmRvYmoKNDkgMCBvYmoKPDwgL0xlbmd0aCAxMzMgL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCnicRY9LDgQhCET3nKKOwMcf53Ey6YVz/+2AnW4TYz2FVIG5gqE9LmsDnRUfIRm28beplo5FWT5UelJWD8ngh6zGyyHcoCzwgkkqhiFQi5gakS1lbreA2zYNsrKVU6WOsIujMI/2tGwVHl+iWyJ1kj+DxCov3OO6Hcil1rveoou+f6QBMQkKZW5kc3RyZWFtCmVuZG9iago1MCAwIG9iago8PCAvTGVuZ3RoIDM0MCAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeJw1UjluBDEM6/0KfSCAbtvv2SBIkfy/DanZFANxdFKUO1pUdsuHhVS17HT5tJXaEjfkd2WFxAnJqxLtUoZIqLxWIdXvmTKvtzVnBMhSpcLkpORxyYI/w6WnC8f5trGv5cgdjx5YFSOhRMAyxcToGpbO7rBmW36WacCPeIScK9Ytx1gFUhvdOO2K96F5LbIGiL2ZlooKHVaJFn5B8aBHjX32GFRYINHtHElwjIlQkYB2gdpIDDl7LHZRH/QzKDET6NobRdxBgSWSmDnFunT03/jQsaD+2Iw3vzoq6VtaWWPSPhvtlMYsMul6WPR089bHgws076L859UMEjRljZLGB63aOYaimVFWeLdDkw3NMcch8w6ewxkJSvo8FL+PJRMdlMjfDg2hf18eo4ycNt4C5qI/bRUHDuKzw165gRVKF2uS9wGpTOiB6f+v8bW+19cfHe2AxgplbmRzdHJlYW0KZW5kb2JqCjUxIDAgb2JqCjw8IC9MZW5ndGggMjUxIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nC1RSXIDQQi7zyv0hGan32OXK4fk/9cIygcGDYtAdFrioIyfICxXvOWRq2jD3zMxgt8Fh34r121Y5EBUIEljUDWhdvF69B7YcZgJzJPWsAxmrA/8jCnc6MXhMRlnt9dl1BDsXa89mUHJrFzEJRMXTNVhI2cOP5kyLrRzPTcg50ZYl2GQblYaMxKONIVIIYWqm6TOBEESjK5GjTZyFPulL490hlWNqDHscy1tX89NOGvQ7Fis8uSUHl1xLicXL6wc9PU2AxdRaazyQEjA/W4P9XOyk994S+fOFtPje83J8sJUYMWb125ANtXi37yI4/uMr+fn+fwDX2BbiAplbmRzdHJlYW0KZW5kb2JqCjUyIDAgb2JqCjw8IC9MZW5ndGggMTc0IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nE2QSQ5DIQxD95zCF6iEM8DnPL+qumjvv61DB3WB/OQgcDw80HEkLnRk6IyOK5sc48CzIGPi0Tj/ybg+xDFB3aItWJd2x9nMEnPCMjECtkbJ2TyiwA/HXAgSZJcfvsAgIl2P+VbzWZP0z7c73Y+6tGZfPaLAiewIxbABV4D9useBS8L5XtPklyolYxOH8oHqIlI2O6EQtVTscqqKs92bK3AV9PzRQ+7tBbUjPN8KZW5kc3RyZWFtCmVuZG9iago1MyAwIG9iago8PCAvTGVuZ3RoIDU0IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4nDM1MFAwUNC1VNA1MjZVMDUEsg3NTBVSDLng7FwIEySfwwVTCWGBpHMQKnO4MrjSAHNRD48KZW5kc3RyZWFtCmVuZG9iago1NCAwIG9iago8PCAvTGVuZ3RoIDE0MSAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeJw9j8EOwzAIQ+/5Cv9ApNgpoXxPp2qH7v+vI0u7C3oCY4yF0NAbqprDhmCb48XSJVRr+BTFQCU3yJlgDqWk0h1HkXpiOBhcHrQbjuKx6PoRu5JmfdDGQrolaIB7rFNp3KZxE8QdNQXqKeqco7wQuZ+pZ9g0kt00s5JzuA2/e89T1/+nq7zL+QW9dy7+CmVuZHN0cmVhbQplbmRvYmoKNTUgMCBvYmoKPDwgL0xlbmd0aCAyMTUgL0ZpbHRlciAvRmxhdGVEZWNvZGUgPj4Kc3RyZWFtCnicNVE5DgMhDOz3Ff5AJIwveE+iKM3+v82M0VYewVyGtJQhmfJSk6gh5VM+epkunLrc18xqNOeWtC1zgLi2vC+tksCJZoiDwWmYuAGaPAFD19GoUUMXHtDUpVMosNwEPoq3bg/dY7WBl7Yh54kgYigZLEHNqUUTFm3PJ6Q1v16LG96X7d3IU6XGlhiBBgFWOBzX6NfwlT1PJtF0FTLUqzXLGAkTRSI8+Y6m1RPrWjTSMhLUxhGsagO8O/0wTgAAE3HLAmSfSpSz5MRvsfSzBlf6/gGfR1SWCmVuZHN0cmVhbQplbmRvYmoKMTUgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL0Jhc2VGb250IC9CTVFRRFYrRGVqYVZ1U2FucyAvRmlyc3RDaGFyIDAgL0xhc3RDaGFyIDI1NQovRm9udERlc2NyaXB0b3IgMTQgMCBSIC9TdWJ0eXBlIC9UeXBlMyAvTmFtZSAvQk1RUURWK0RlamFWdVNhbnMKL0ZvbnRCQm94IFsgLTEwMjEgLTQ2MyAxNzk0IDEyMzMgXSAvRm9udE1hdHJpeCBbIDAuMDAxIDAgMCAwLjAwMSAwIDAgXQovQ2hhclByb2NzIDE2IDAgUgovRW5jb2RpbmcgPDwgL1R5cGUgL0VuY29kaW5nCi9EaWZmZXJlbmNlcyBbIDMyIC9zcGFjZSA0NiAvcGVyaW9kIDQ4IC96ZXJvIC9vbmUgL3R3byAvdGhyZWUgL2ZvdXIgL2ZpdmUgL3NpeCA1NgovZWlnaHQgL25pbmUgNjggL0QgL0UgL0YgL0cgNzMgL0kgNzcgL00gL04gODMgL1MgL1QgODYgL1YgOTUgL3VuZGVyc2NvcmUgOTcKL2EgOTkgL2MgL2QgL2UgL2YgMTA0IC9oIC9pIDEwOCAvbCAvbSAvbiAvbyAvcCAxMTQgL3IgL3MgL3QgL3UgMTIxIC95IF0KPj4KL1dpZHRocyAxMyAwIFIgPj4KZW5kb2JqCjE0IDAgb2JqCjw8IC9UeXBlIC9Gb250RGVzY3JpcHRvciAvRm9udE5hbWUgL0JNUVFEVitEZWphVnVTYW5zIC9GbGFncyAzMgovRm9udEJCb3ggWyAtMTAyMSAtNDYzIDE3OTQgMTIzMyBdIC9Bc2NlbnQgOTI5IC9EZXNjZW50IC0yMzYgL0NhcEhlaWdodCAwCi9YSGVpZ2h0IDAgL0l0YWxpY0FuZ2xlIDAgL1N0ZW1WIDAgL01heFdpZHRoIDEzNDIgPj4KZW5kb2JqCjEzIDAgb2JqClsgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAKNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCAzMTggNDAxIDQ2MCA4MzggNjM2Cjk1MCA3ODAgMjc1IDM5MCAzOTAgNTAwIDgzOCAzMTggMzYxIDMxOCAzMzcgNjM2IDYzNiA2MzYgNjM2IDYzNiA2MzYgNjM2IDYzNgo2MzYgNjM2IDMzNyAzMzcgODM4IDgzOCA4MzggNTMxIDEwMDAgNjg0IDY4NiA2OTggNzcwIDYzMiA1NzUgNzc1IDc1MiAyOTUKMjk1IDY1NiA1NTcgODYzIDc0OCA3ODcgNjAzIDc4NyA2OTUgNjM1IDYxMSA3MzIgNjg0IDk4OSA2ODUgNjExIDY4NSAzOTAgMzM3CjM5MCA4MzggNTAwIDUwMCA2MTMgNjM1IDU1MCA2MzUgNjE1IDM1MiA2MzUgNjM0IDI3OCAyNzggNTc5IDI3OCA5NzQgNjM0IDYxMgo2MzUgNjM1IDQxMSA1MjEgMzkyIDYzNCA1OTIgODE4IDU5MiA1OTIgNTI1IDYzNiAzMzcgNjM2IDgzOCA2MDAgNjM2IDYwMCAzMTgKMzUyIDUxOCAxMDAwIDUwMCA1MDAgNTAwIDEzNDIgNjM1IDQwMCAxMDcwIDYwMCA2ODUgNjAwIDYwMCAzMTggMzE4IDUxOCA1MTgKNTkwIDUwMCAxMDAwIDUwMCAxMDAwIDUyMSA0MDAgMTAyMyA2MDAgNTI1IDYxMSAzMTggNDAxIDYzNiA2MzYgNjM2IDYzNiAzMzcKNTAwIDUwMCAxMDAwIDQ3MSA2MTIgODM4IDM2MSAxMDAwIDUwMCA1MDAgODM4IDQwMSA0MDEgNTAwIDYzNiA2MzYgMzE4IDUwMAo0MDEgNDcxIDYxMiA5NjkgOTY5IDk2OSA1MzEgNjg0IDY4NCA2ODQgNjg0IDY4NCA2ODQgOTc0IDY5OCA2MzIgNjMyIDYzMiA2MzIKMjk1IDI5NSAyOTUgMjk1IDc3NSA3NDggNzg3IDc4NyA3ODcgNzg3IDc4NyA4MzggNzg3IDczMiA3MzIgNzMyIDczMiA2MTEgNjA1CjYzMCA2MTMgNjEzIDYxMyA2MTMgNjEzIDYxMyA5ODIgNTUwIDYxNSA2MTUgNjE1IDYxNSAyNzggMjc4IDI3OCAyNzggNjEyIDYzNAo2MTIgNjEyIDYxMiA2MTIgNjEyIDgzOCA2MTIgNjM0IDYzNCA2MzQgNjM0IDU5MiA2MzUgNTkyIF0KZW5kb2JqCjE2IDAgb2JqCjw8IC9EIDE3IDAgUiAvRSAxOCAwIFIgL0YgMTkgMCBSIC9HIDIwIDAgUiAvSSAyMSAwIFIgL00gMjIgMCBSIC9OIDIzIDAgUgovUyAyNCAwIFIgL1QgMjUgMCBSIC9WIDI2IDAgUiAvYSAyNyAwIFIgL2MgMjggMCBSIC9kIDI5IDAgUiAvZSAzMCAwIFIKL2VpZ2h0IDMxIDAgUiAvZiAzMiAwIFIgL2ZpdmUgMzMgMCBSIC9mb3VyIDM0IDAgUiAvaCAzNSAwIFIgL2kgMzYgMCBSCi9sIDM3IDAgUiAvbSAzOCAwIFIgL24gMzkgMCBSIC9uaW5lIDQwIDAgUiAvbyA0MSAwIFIgL29uZSA0MiAwIFIgL3AgNDMgMCBSCi9wZXJpb2QgNDQgMCBSIC9yIDQ1IDAgUiAvcyA0NiAwIFIgL3NpeCA0NyAwIFIgL3NwYWNlIDQ4IDAgUiAvdCA0OSAwIFIKL3RocmVlIDUwIDAgUiAvdHdvIDUxIDAgUiAvdSA1MiAwIFIgL3VuZGVyc2NvcmUgNTMgMCBSIC95IDU0IDAgUgovemVybyA1NSAwIFIgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL0YxIDE1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvQTEgPDwgL1R5cGUgL0V4dEdTdGF0ZSAvQ0EgMCAvY2EgMSA+PgovQTIgPDwgL1R5cGUgL0V4dEdTdGF0ZSAvQ0EgMSAvY2EgMSA+PgovQTMgPDwgL1R5cGUgL0V4dEdTdGF0ZSAvQ0EgMC44IC9jYSAwLjggPj4gPj4KZW5kb2JqCjUgMCBvYmoKPDwgPj4KZW5kb2JqCjYgMCBvYmoKPDwgPj4KZW5kb2JqCjcgMCBvYmoKPDwgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFsgMTEgMCBSIF0gL0NvdW50IDEgPj4KZW5kb2JqCjU2IDAgb2JqCjw8IC9DcmVhdG9yIChNYXRwbG90bGliIHYzLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZykKL1Byb2R1Y2VyIChNYXRwbG90bGliIHBkZiBiYWNrZW5kIHYzLjcuMSkgL0NyZWF0aW9uRGF0ZSAoRDoyMDIzMDMxNTEwMDE1OFopCj4+CmVuZG9iagp4cmVmCjAgNTcKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDE2IDAwMDAwIG4gCjAwMDAwMTQ2NzQgMDAwMDAgbiAKMDAwMDAxNDQzNyAwMDAwMCBuIAowMDAwMDE0NDY5IDAwMDAwIG4gCjAwMDAwMTQ2MTEgMDAwMDAgbiAKMDAwMDAxNDYzMiAwMDAwMCBuIAowMDAwMDE0NjUzIDAwMDAwIG4gCjAwMDAwMDAwNjUgMDAwMDAgbiAKMDAwMDAwMDM0MSAwMDAwMCBuIAowMDAwMDAyNDMzIDAwMDAwIG4gCjAwMDAwMDAyMDggMDAwMDAgbiAKMDAwMDAwMjQxMiAwMDAwMCBuIAowMDAwMDEyOTI4IDAwMDAwIG4gCjAwMDAwMTI3MjEgMDAwMDAgbiAKMDAwMDAxMjIwMSAwMDAwMCBuIAowMDAwMDEzOTgxIDAwMDAwIG4gCjAwMDAwMDI0NTMgMDAwMDAgbiAKMDAwMDAwMjY5MCAwMDAwMCBuIAowMDAwMDAyODQzIDAwMDAwIG4gCjAwMDAwMDI5OTEgMDAwMDAgbiAKMDAwMDAwMzMxMSAwMDAwMCBuIAowMDAwMDAzNDM0IDAwMDAwIG4gCjAwMDAwMDM1OTYgMDAwMDAgbiAKMDAwMDAwMzc0NSAwMDAwMCBuIAowMDAwMDA0MTU5IDAwMDAwIG4gCjAwMDAwMDQyOTcgMDAwMDAgbiAKMDAwMDAwNDQ0MSAwMDAwMCBuIAowMDAwMDA0ODIxIDAwMDAwIG4gCjAwMDAwMDUxMjYgMDAwMDAgbiAKMDAwMDAwNTQzMCAwMDAwMCBuIAowMDAwMDA1NzUyIDAwMDAwIG4gCjAwMDAwMDYyMjAgMDAwMDAgbiAKMDAwMDAwNjQyOSAwMDAwMCBuIAowMDAwMDA2NzUxIDAwMDAwIG4gCjAwMDAwMDY5MTcgMDAwMDAgbiAKMDAwMDAwNzE1NCAwMDAwMCBuIAowMDAwMDA3Mjk4IDAwMDAwIG4gCjAwMDAwMDc0MTcgMDAwMDAgbiAKMDAwMDAwNzc0OCAwMDAwMCBuIAowMDAwMDA3OTg0IDAwMDAwIG4gCjAwMDAwMDgzNzkgMDAwMDAgbiAKMDAwMDAwODY3MCAwMDAwMCBuIAowMDAwMDA4ODI1IDAwMDAwIG4gCjAwMDAwMDkxMzcgMDAwMDAgbiAKMDAwMDAwOTI2MCAwMDAwMCBuIAowMDAwMDA5NDkzIDAwMDAwIG4gCjAwMDAwMDk5MDAgMDAwMDAgbiAKMDAwMDAxMDI5MyAwMDAwMCBuIAowMDAwMDEwMzgzIDAwMDAwIG4gCjAwMDAwMTA1ODkgMDAwMDAgbiAKMDAwMDAxMTAwMiAwMDAwMCBuIAowMDAwMDExMzI2IDAwMDAwIG4gCjAwMDAwMTE1NzMgMDAwMDAgbiAKMDAwMDAxMTY5OSAwMDAwMCBuIAowMDAwMDExOTEzIDAwMDAwIG4gCjAwMDAwMTQ3MzQgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA1NyAvUm9vdCAxIDAgUiAvSW5mbyA1NiAwIFIgPj4Kc3RhcnR4cmVmCjE0ODg1CiUlRU9GCg==", "image/svg+xml": ["\n", "\n", "\n"], "text/plain": ["
![](\"https://github.com/Lightning-AI/lightning-tutorials/raw/main/course_UvA-DL/03-initialization-and-optimization/flat_vs_sharp_minima.svg\")