diff --git a/.gitignore b/.gitignore index ceaa24ba..053d0af9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,9 @@ __pycache__ *~ *.npy _build -model_1 -model_2 -model_3 .DS_Store +.claude/ +hls4ml_prjs/ +data/ +models/ +6_more_models/outputs/ diff --git a/1_getting_started/1a_train_keras.ipynb b/1_getting_started/1a_train_keras.ipynb new file mode 100644 index 00000000..49c8e9ef --- /dev/null +++ b/1_getting_started/1a_train_keras.ipynb @@ -0,0 +1,263 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Part 1a: Getting started with Keras\n", + "\n", + "In this notebook we will train a small neural network on the LHC jet tagging dataset using Keras v3. When you are done, head straight to **`1c_hls4ml_synth.ipynb`** to convert the trained model to an FPGA design with hls4ml." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48fc9aa8", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['KERAS_BACKEND'] = 'tensorflow'\n", + "\n", + "import numpy as np\n", + "import sys\n", + "\n", + "sys.path.append('..')\n", + "\n", + "from sklearn.datasets import fetch_openml\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import LabelEncoder, StandardScaler\n", + "\n", + "%matplotlib inline\n", + "np.random.seed(0)" + ] + }, + { + "cell_type": "markdown", + "id": "1ea16a9e", + "metadata": {}, + "source": [ + "## Fetch the jet tagging dataset from Open ML\n", + "\n", + "The [HLS4ML LHC jet dataset](https://openml.org/search?type=data&id=42468) was introduced by [Duarte et al. (2018)](https://arxiv.org/abs/1804.06913) to benchmark fast neural network inference on FPGAs for particle physics applications.\n", + "\n", + "Jets are collimated sprays of particles produced when quarks or gluons are knocked out of colliding protons at the LHC. Identifying the origin of a jet in real time is a core task for LHC trigger systems, which must decide within a few microseconds whether to keep or discard each collision event.\n", + "\n", + "The dataset contains 16 high-level jet substructure observables derived from simulated proton-proton collisions at √s = 13 TeV. These include energy correlation functions, N-subjettiness ratios, a groomed jet mass, and constituent multiplicity. The goal is to classify each jet into one of five categories:\n", + "\n", + "| Label | Jet origin |\n", + "|-------|------------|\n", + "| `g` | Gluon |\n", + "| `q` | Light quark |\n", + "| `w` | W boson decay (W → qq') |\n", + "| `z` | Z boson decay (Z → qq') |\n", + "| `t` | Top quark decay (t → bqq') |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4c7e8da3", + "metadata": {}, + "outputs": [], + "source": [ + "data = fetch_openml('hls4ml_lhc_jets_hlf')\n", + "X, y = data['data'], data['target']" + ] + }, + { + "cell_type": "markdown", + "id": "77c862eb", + "metadata": {}, + "source": [ + "### Let's print some information about the dataset\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77e2e0b9", + "metadata": {}, + "outputs": [], + "source": [ + "print(data['feature_names'])\n", + "print(X.shape, y.shape)\n", + "print(X[:5])\n", + "print(y[:5])" + ] + }, + { + "cell_type": "markdown", + "id": "41b77e9a", + "metadata": {}, + "source": [ + "As you saw above, the `y` target is an array of strings, e.g. `['g', 'w', ...]` etc.\n", + "We need to make this a \"One Hot\" encoding for the training.\n", + "Then, split the dataset into training and validation sets:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31201326", + "metadata": {}, + "outputs": [], + "source": [ + "le = LabelEncoder()\n", + "y_encoded = le.fit_transform(y)\n", + "y = np.eye(5)[y_encoded] # one-hot encode\n", + "X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n", + "print(y[:5])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3b4d39e", + "metadata": {}, + "outputs": [], + "source": "scaler = StandardScaler()\nX_train_val = scaler.fit_transform(X_train_val)\nX_test = scaler.transform(X_test)\n\nos.makedirs('../data/jet-tagging', exist_ok=True)\nnp.save('../data/jet-tagging/X_train_val.npy', X_train_val)\nnp.save('../data/jet-tagging/X_test.npy', X_test)\nnp.save('../data/jet-tagging/y_train_val.npy', y_train_val)\nnp.save('../data/jet-tagging/y_test.npy', y_test)\nnp.save('../data/jet-tagging/classes.npy', le.classes_)" + }, + { + "cell_type": "markdown", + "id": "f9238a75", + "metadata": {}, + "source": [ + "## Now construct a model\n", + "We'll use 3 hidden layers with 64, then 32, then 32 neurons. Each layer will use ReLU activation.\n", + "Finally, we add an output layer with 5 neurons and the Softmax activation, to calculate the probability of each of the five classes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c50f7cd", + "metadata": {}, + "outputs": [], + "source": [ + "from keras.models import Sequential\n", + "from keras.layers import Dense\n", + "from keras.optimizers import Adam\n", + "\n", + "model = Sequential()\n", + "model.add(Dense(64, input_shape=(16,), name='fc1', activation='relu'))\n", + "model.add(Dense(32, name='fc2', activation='relu'))\n", + "model.add(Dense(32, name='fc3', activation='relu'))\n", + "model.add(Dense(5, name='output', activation='softmax'))\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "id": "da38ba67", + "metadata": {}, + "source": [ + "## Train the model\n", + "We'll use the Adam optimiser with categorical crossentropy loss.\n", + "The model isn't very complex, so this should take just a few minutes even on a CPU." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "520e2fc5", + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(optimizer=Adam(learning_rate=1e-3), loss='categorical_crossentropy', metrics=['accuracy'])\n", + "model.fit(\n", + " X_train_val,\n", + " y_train_val,\n", + " batch_size=1024,\n", + " epochs=20,\n", + " validation_split=0.25,\n", + " shuffle=True,\n", + ")\n", + "os.makedirs('../models', exist_ok=True)\n", + "model.save('../models/keras_model_part1.h5')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Check performance\n", + "Check the accuracy and make a ROC curve:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import plotting\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "y_keras = model.predict(X_test)\n", + "print(\"Accuracy: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_keras, axis=1))))\n", + "plt.figure(figsize=(9, 9))\n", + "_ = plotting.makeRoc(y_test, y_keras, le.classes_)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An accuracy of ~75% is expected for this 5-class problem — random guessing gives only 20%, and some classes (notably gluon vs. light quark) are physically very similar and genuinely hard to separate even with more sophisticated methods.\n", + "\n", + "The ROC (Receiver Operating Characteristic) curve shows, for each class, the trade-off between signal efficiency (true positive rate) and background efficiency (false positive rate) as the decision threshold is varied. The area under the curve (AUC) ranges from 0.5 (random classifier) to 1.0 (perfect). Higher and further to the upper-left is better." + ] + }, + { + "cell_type": "markdown", + "id": "713edc21", + "metadata": {}, + "source": [ + "**N.B.** This notebook trains a full-precision (32-bit floating-point) model. When converting to an FPGA design, hls4ml applies post-training quantization (PTQ) by default, which works well at 16-bit precision but struggles to match accuracy below ~8 bits. For the most resource-efficient FPGA designs **quantization-aware training (QAT)** gives substantially better results. See **Part 2** for QKeras (Keras) and Brevitas (PyTorch) QAT examples." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next step\n", + "\n", + "Your model is trained and saved. Open **`1c_hls4ml_synth.ipynb`** to convert it to an FPGA design with hls4ml." + ] + }, + { + "cell_type": "markdown", + "id": "1c38f1d6", + "metadata": {}, + "source": [ + "## Further reading\n", + "\n", + "For more details, see: Duarte, Han, Harris et al., \"Fast inference of deep neural networks in FPGAs for particle physics\", JINST 13 P07027 (2018), [arXiv:1804.06913](https://arxiv.org/abs/1804.06913)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "hls4ml-tutorial", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/1_getting_started/1b_train_pytorch.ipynb b/1_getting_started/1b_train_pytorch.ipynb new file mode 100644 index 00000000..596f09e3 --- /dev/null +++ b/1_getting_started/1b_train_pytorch.ipynb @@ -0,0 +1,284 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Part 1b: Getting started with PyTorch\n", + "\n", + "In this notebook we train the same network architecture on the LHC jet tagging dataset using PyTorch. When you are done, head straight to **`1c_hls4ml_synth.ipynb`** to convert the model to an FPGA design." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import os\n", + "import sys\n", + "\n", + "sys.path.append('..')\n", + "\n", + "from sklearn.datasets import fetch_openml\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import LabelEncoder, StandardScaler\n", + "\n", + "%matplotlib inline\n", + "np.random.seed(0)\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "from torch.utils.data import DataLoader, TensorDataset\n", + "\n", + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Fetch the jet tagging dataset from Open ML\n", + "\n", + "The [HLS4ML LHC jet dataset](https://openml.org/search?type=data&id=42468) was introduced in [Duarte et al. (2018)](https://arxiv.org/abs/1804.06913) to benchmark fast neural network inference on FPGAs for particle physics applications.\n", + "\n", + "Jets are collimated sprays of particles produced when quarks or gluons are knocked out of colliding protons at the LHC. Identifying the origin of a jet in real time is a core task for LHC trigger systems, which must decide within a few microseconds whether to keep or discard each collision event.\n", + "\n", + "The dataset contains 16 high-level jet substructure observables derived from simulated proton-proton collisions at √s = 13 TeV. These include energy correlation functions, N-subjettiness ratios, a groomed jet mass, and constituent multiplicity. The goal is to classify each jet into one of five categories:\n", + "\n", + "| Label | Jet origin |\n", + "|-------|------------|\n", + "| `g` | Gluon |\n", + "| `q` | Light quark |\n", + "| `w` | W boson decay (W → qq') |\n", + "| `z` | Z boson decay (Z → qq') |\n", + "| `t` | Top quark decay (t → bqq') |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = fetch_openml('hls4ml_lhc_jets_hlf')\n", + "X, y = data['data'], data['target']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Let's print some information about the dataset\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(data['feature_names'])\n", + "print(X.shape, y.shape)\n", + "print(X[:5])\n", + "print(y[:5])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you saw above, the `y` target is an array of strings, e.g. `['g', 'w', ...]` etc.\n", + "We need to make this a \"One Hot\" encoding for the training.\n", + "Then, split the dataset into training and validation sets:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "le = LabelEncoder()\n", + "y_encoded = le.fit_transform(y)\n", + "y = np.eye(5)[y_encoded] # one-hot encode\n", + "X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n", + "print(y[:5])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "scaler = StandardScaler()\nX_train_val = scaler.fit_transform(X_train_val)\nX_test = scaler.transform(X_test)\n\nos.makedirs('../data/jet-tagging', exist_ok=True)\nnp.save('../data/jet-tagging/X_train_val.npy', X_train_val)\nnp.save('../data/jet-tagging/X_test.npy', X_test)\nnp.save('../data/jet-tagging/y_train_val.npy', y_train_val)\nnp.save('../data/jet-tagging/y_test.npy', y_test)\nnp.save('../data/jet-tagging/classes.npy', le.classes_)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Now construct a model\n", + "We'll use 3 hidden layers with 64, then 32, then 32 neurons with ReLU activation, and a 5-neuron output with Softmax.\n", + "\n", + "Note: unlike Keras, PyTorch's `CrossEntropyLoss` fuses LogSoftmax and NLLLoss internally and therefore expects raw logits. Because Softmax is part of our model, we instead use `NLLLoss` with the log of the model output, which is equivalent." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class JetTagger(nn.Module):\n", + " \"\"\"Simple 3-hidden-layer jet tagger: 16 → 64 → 32 → 32 → 5.\"\"\"\n", + "\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(16, 64)\n", + " self.fc2 = nn.Linear(64, 32)\n", + " self.fc3 = nn.Linear(32, 32)\n", + " self.output = nn.Linear(32, 5)\n", + "\n", + " def forward(self, x):\n", + " x = torch.relu(self.fc1(x))\n", + " x = torch.relu(self.fc2(x))\n", + " x = torch.relu(self.fc3(x))\n", + " return torch.softmax(self.output(x), dim=1)\n", + "\n", + "\n", + "model = JetTagger()\n", + "print(model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Train the model\n", + "We'll use the Adam optimiser with NLL loss.\n", + "The model isn't very complex, so this should take just a few minutes even on the CPU." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6373c9e4", + "metadata": {}, + "outputs": [], + "source": [ + "n_train = int(len(X_train_val) * 0.75)\n", + "\n", + "X_tr = torch.FloatTensor(X_train_val[:n_train])\n", + "y_tr = torch.LongTensor(np.argmax(y_train_val[:n_train], axis=1))\n", + "X_val = torch.FloatTensor(X_train_val[n_train:])\n", + "y_val = torch.LongTensor(np.argmax(y_train_val[n_train:], axis=1))\n", + "\n", + "loader = DataLoader(TensorDataset(X_tr, y_tr), batch_size=1024, shuffle=True)\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n", + "criterion = nn.NLLLoss()\n", + "\n", + "for epoch in range(20):\n", + " model.train()\n", + " for X_batch, y_batch in loader:\n", + " optimizer.zero_grad()\n", + " loss = criterion(torch.log(model(X_batch).clamp(min=1e-7)), y_batch)\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " model.eval()\n", + " with torch.no_grad():\n", + " val_loss = criterion(torch.log(model(X_val).clamp(min=1e-7)), y_val).item()\n", + " print(f'Epoch {epoch + 1:2d} val_loss={val_loss:.4f}')\n", + "\n", + "os.makedirs('../models', exist_ok=True)\n", + "torch.save(model.state_dict(), '../models/pytorch_weights_part1.pt')\n", + "print('Saved ../models/pytorch_weights_part1.pt')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Check performance\n", + "Check the accuracy and make a ROC curve:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import plotting\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "model.eval()\n", + "with torch.no_grad():\n", + " y_pytorch = model(torch.FloatTensor(X_test)).numpy()\n", + "\n", + "print(\"Accuracy: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_pytorch, axis=1))))\n", + "plt.figure(figsize=(9, 9))\n", + "_ = plotting.makeRoc(y_test, y_pytorch, le.classes_)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An accuracy of ~75% is expected for this 5-class problem — random guessing gives only 20%, and some classes (notably gluon vs. light quark) are physically very similar and genuinely hard to separate even with more sophisticated methods.\n", + "\n", + "The ROC (Receiver Operating Characteristic) curve shows, for each class, the trade-off between signal efficiency (true positive rate) and background efficiency (false positive rate) as the decision threshold is varied. The area under the curve (AUC) ranges from 0.5 (random classifier) to 1.0 (perfect). Higher and further to the upper-left is better." + ] + }, + { + "cell_type": "markdown", + "id": "c45ffa20", + "metadata": {}, + "source": [ + "**N.B.** This notebook trains a full-precision (32-bit floating-point) model. When converting to an FPGA design, hls4ml applies post-training quantization (PTQ) by default, which works well at 16-bit precision but struggles to match accuracy below ~8 bits. For the most resource-efficient FPGA designs, **quantization-aware training (QAT)** gives substantially better results. See **Part 2** for QKeras (Keras) and Brevitas (PyTorch) QAT examples." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next step\n", + "\n", + "Your model is trained and saved. Open **`1c_hls4ml_synth.ipynb`** to convert it to an FPGA design with hls4ml." + ] + }, + { + "cell_type": "markdown", + "id": "c2603bd5", + "metadata": {}, + "source": [ + "## Further reading\n", + "\n", + "For more details, see: Duarte, Han, Harris et al., \"Fast inference of deep neural networks in FPGAs for particle physics\", JINST 13 P07027 (2018), [arXiv:1804.06913](https://arxiv.org/abs/1804.06913)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "hls4ml-tutorial", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/1_getting_started/1c_hls4ml_synth.ipynb b/1_getting_started/1c_hls4ml_synth.ipynb new file mode 100644 index 00000000..e433e118 --- /dev/null +++ b/1_getting_started/1c_hls4ml_synth.ipynb @@ -0,0 +1,335 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Part 1c: Converting to an FPGA design with hls4ml\n", + "\n", + "Now we will go through the steps to convert the model we trained to a low-latency FPGA design with hls4ml.\n", + "First, we will evaluate its classification performance to make sure we haven't lost accuracy using fixed-point data types.\n", + "Then we will synthesize the model with Vitis HLS and check the latency and FPGA resource usage.\n", + "\n", + "Run either `1a_train_keras.ipynb` or `1b_train_pytorch.ipynb` first, then set `MODEL_TYPE` in the cell below accordingly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "MODEL_TYPE = 'keras' # set to 'pytorch' if you used the PyTorch notebook" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['KERAS_BACKEND'] = 'tensorflow'\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import sys\n", + "\n", + "sys.path.append('..')\n", + "import plotting\n", + "import hls4ml\n", + "\n", + "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']\n", + "\n", + "# Load the test dataset\n", + "X_test = np.ascontiguousarray(np.load('../data/jet-tagging/X_test.npy'), dtype=np.float32)\n", + "y_test = np.load('../data/jet-tagging/y_test.npy')\n", + "classes = np.load('../data/jet-tagging/classes.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load the trained model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b8c143ac", + "metadata": {}, + "outputs": [], + "source": [ + "if MODEL_TYPE == 'keras':\n", + " import tensorflow as tf\n", + "\n", + " trained_model = tf.keras.models.load_model('../models/keras_model_part1.h5')\n", + " y_model = trained_model.predict(X_test)\n", + "\n", + "elif MODEL_TYPE == 'pytorch':\n", + " import torch\n", + " import torch.nn as nn\n", + "\n", + " # Architecture must match 1b_train_pytorch.ipynb\n", + " class JetTagger(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(16, 64)\n", + " self.fc2 = nn.Linear(64, 32)\n", + " self.fc3 = nn.Linear(32, 32)\n", + " self.output = nn.Linear(32, 5)\n", + "\n", + " def forward(self, x):\n", + " x = torch.relu(self.fc1(x))\n", + " x = torch.relu(self.fc2(x))\n", + " x = torch.relu(self.fc3(x))\n", + " return torch.softmax(self.output(x), dim=1)\n", + "\n", + " trained_model = JetTagger()\n", + " trained_model.load_state_dict(torch.load('../models/pytorch_weights_part1.pt'))\n", + " trained_model.eval()\n", + " with torch.no_grad():\n", + " y_model = trained_model(torch.FloatTensor(X_test)).numpy()\n", + "\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "print(\"Accuracy: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_model, axis=1))))" + ] + }, + { + "cell_type": "markdown", + "id": "1599cdfe", + "metadata": {}, + "source": [ + "## Make an hls4ml config & model\n", + "\n", + "The hls4ml configuration is a dictionary that controls how the model is mapped to FPGA hardware. It exposes several knobs to tune the trade-off between latency, resource usage, and model accuracy:\n", + "\n", + "- **Precision**: the fixed-point bit-width used for weights, biases, and accumulators. Reducing precision saves resources but can degrade accuracy unless the model was trained with quantization in mind. Covered in more details in **Part 2**.\n", + "- **Reuse factor**: controls parallelisation. A reuse factor of 1 gives maximum parallelism (lowest latency, most resources); higher values reuse the same hardware across multiple operations, saving resources at the cost of latency. Covered in **Part 3**.\n", + "- **I/O type**: controls how data is streamed through the network, which is particularly important for CNNs. Covered in **Part 4**.\n", + "- **Strategy**: selects the implementation algorithm, e.g. `Latency` or `Resource`.\n", + "\n", + "In this part, we use the default hls4ml configuration to get a baseline design for the FPGA." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "465985cd", + "metadata": {}, + "outputs": [], + "source": [ + "if MODEL_TYPE == 'keras':\n", + " config = hls4ml.utils.config_from_keras_model(trained_model, granularity='model', backend='Vitis')\n", + " hls_model = hls4ml.converters.convert_from_keras_model(\n", + " trained_model,\n", + " hls_config=config,\n", + " backend='Vitis',\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_base_part1',\n", + " part='xcu200-fsgd2104-2-e',\n", + " )\n", + "\n", + "elif MODEL_TYPE == 'pytorch':\n", + " config = hls4ml.utils.config_from_pytorch_model(\n", + " trained_model,\n", + " input_shape=(16,),\n", + " granularity='model',\n", + " backend='Vitis',\n", + " )\n", + " hls_model = hls4ml.converters.convert_from_pytorch_model(\n", + " trained_model,\n", + " hls_config=config,\n", + " backend='Vitis',\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_base_part1',\n", + " part='xcu200-fsgd2104-2-e',\n", + " )\n", + "\n", + "print(\"-----------------------------------\")\n", + "print(\"Configuration\")\n", + "plotting.print_dict(config)\n", + "print(\"-----------------------------------\")" + ] + }, + { + "cell_type": "markdown", + "id": "da5414e7", + "metadata": {}, + "source": [ + "Let's visualise what we created. The model architecture is shown, annotated with the shape and data types." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d2a1d83", + "metadata": {}, + "outputs": [], + "source": [ + "hls4ml.utils.plot_model(hls_model, show_shapes=True, show_precision=True, to_file=None)" + ] + }, + { + "cell_type": "markdown", + "id": "46eb6a16", + "metadata": {}, + "source": [ + "## Compile & predict\n", + "\n", + "hls4ml uses **fixed-point arithmetic** for everything on the FPGA — weights, biases, activations, and accumulators — as a form of post-training quantization. The default precision is `ap_fixed<16,6>` (16 bits total, 6 integer bits). This is in contrast to the CPU, which uses 32-bit floating-point throughout.\n", + "\n", + "To confirm that this quantization has not degraded accuracy, we compile the hls4ml model and run `hls_model.predict`, which emulates the fixed-point FPGA design bit-accurately on the CPU. If the ROC curves match closely, the model performance is well-preserved after quantization.\n", + "\n", + "Note that with the default 16-bit precision, accuracy is usually maintained without any special treatment. However, lower precisions and better resource savings can be achieved through *quantization-aware training* (QAT), which is further explored in **Part 2**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "add39e21", + "metadata": {}, + "outputs": [], + "source": [ + "hls_model.compile()\n", + "y_hls = hls_model.predict(X_test)" + ] + }, + { + "cell_type": "markdown", + "id": "46ed2f44", + "metadata": {}, + "source": [ + "## Compare\n", + "That was easy! Now let's see how the performance compares to the original model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9054d60", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "\n", + "print(\"{} Accuracy: {}\".format(MODEL_TYPE, accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_model, axis=1))))\n", + "print(\"hls4ml Accuracy: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_hls, axis=1))))\n", + "\n", + "fig, ax = plt.subplots(figsize=(9, 9))\n", + "_ = plotting.makeRoc(y_test, y_model, list(classes))\n", + "plt.gca().set_prop_cycle(None)\n", + "_ = plotting.makeRoc(y_test, y_hls, list(classes), linestyle='--')\n", + "\n", + "from matplotlib.lines import Line2D\n", + "from matplotlib.legend import Legend\n", + "\n", + "lines = [Line2D([0], [0], ls='-'), Line2D([0], [0], ls='--')]\n", + "leg = Legend(ax, lines, labels=[MODEL_TYPE, 'hls4ml'], loc='lower right', frameon=False)\n", + "ax.add_artist(leg)" + ] + }, + { + "cell_type": "markdown", + "id": "c322837a", + "metadata": {}, + "source": [ + "## Synthesize\n", + "Now we'll actually use Vitis HLS to synthesize the model. Vitis HLS compiles the HLS project into RTL, which can be exported as an FPGA IP and interated with a larger design, deployed on the FPGA.\n", + "\n", + "In this tutorial, we will only review the HLS report, checking the latency and resource consumption.\n", + "\n", + "**This can take several minutes.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56eacd3d", + "metadata": {}, + "outputs": [], + "source": [ + "hls_model.build(csim=False)" + ] + }, + { + "cell_type": "markdown", + "id": "6b2b9c92", + "metadata": {}, + "source": [ + "# Check the reports\n", + "Print out the reports generated by Vitis HLS. Pay attention to the Latency and the 'Utilization Estimates' sections." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ebab7c18", + "metadata": {}, + "outputs": [], + "source": [ + "hls4ml.report.read_vivado_report('../hls4ml_prjs/hls4ml_prj_base_part1')" + ] + }, + { + "cell_type": "markdown", + "id": "392d6b01", + "metadata": {}, + "source": [ + "## A note on DSP usage\n", + "\n", + "You may notice that the baseline design uses a significant number of **DSPs** (Digital Signal Processing blocks). This is because the default weight and activation precision is `ap_fixed<16,6>` — 16-bit fixed-point. Generally, multiplications with precisions wider than 10 bits are mapped to DSPs, while narrower multiplications are implemented with LUTs.\n", + "\n", + "FPGAs typically have far more LUTs than DSPs — the Alveo U250, for example, has ~1.7 million LUTs but only ~12,288 DSPs; limiting the scale of our design.\n", + "\n", + "The easiest way to reduce DSP consumption is by reducing variable precision through **quantization-aware training (QAT)**, which is covered in more details in **Part 2** with QKeras and Brevitas." + ] + }, + { + "cell_type": "markdown", + "id": "ae257dd1", + "metadata": {}, + "source": [ + "# A note on resource estimates\n", + "\n", + "The resource numbers reported by Vitis HLS after HLS C-synthesis are **estimates** derived from the HLS's internal model and often do not truly reflect the final resource consumption of the model. These estimates **often overestimate** LUT consumption, sometimes by an order of magnitude.\n", + "\n", + "For a more accurate picture of resource consumption, you should run **Vivado synthesis** (`vsynth`). This invokes the full Vivado synthesis flow on the generated RTL, producing estimates that are much closer to what you would see after implementation (place-and-route).\n", + "\n", + "**This step can take 10–20 minutes.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "226b890b", + "metadata": {}, + "outputs": [], + "source": [ + "hls_model.build(reset=False, csim=False, vsynth=True)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "hls4ml-tutorial", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/2_quantization/2a_qkeras.ipynb b/2_quantization/2a_qkeras.ipynb new file mode 100644 index 00000000..00049be3 --- /dev/null +++ b/2_quantization/2a_qkeras.ipynb @@ -0,0 +1,367 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# Part 2a: Quantization with QKeras\n", + "\n", + "In this notebook we retrain the jet tagger from Part 1 using **QKeras** (Quantized Keras). With quantization-aware training (QAT), the model is trained with low-precision, fixed-point weights, so the optimizer can correct the effect of quantization during training rather than after — enabling lower precisions (and thus resource consumption) without sacrificing accuracy.\n", + "\n", + "Make sure you have run `1_getting_started/1a_train_keras.ipynb` first, as we load its saved data and use its model as a baseline for comparison." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['KERAS_BACKEND'] = 'tensorflow'\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import sys\n", + "\n", + "sys.path.append('..')\n", + "import plotting\n", + "\n", + "%matplotlib inline\n", + "seed = 0\n", + "np.random.seed(seed)\n", + "import tensorflow as tf\n", + "\n", + "tf.random.set_seed(seed)\n", + "\n", + "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']" + ] + }, + { + "cell_type": "markdown", + "id": "cell-2", + "metadata": {}, + "source": [ + "## Load the jet tagging dataset\n", + "\n", + "We load the preprocessed arrays saved by `1a_train_keras.ipynb`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-3", + "metadata": {}, + "outputs": [], + "source": [ + "X_train_val = np.load('../data/jet-tagging/X_train_val.npy')\n", + "X_test = np.load('../data/jet-tagging/X_test.npy')\n", + "y_train_val = np.load('../data/jet-tagging/y_train_val.npy')\n", + "y_test = np.load('../data/jet-tagging/y_test.npy')\n", + "classes = np.load('../data/jet-tagging/classes.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-4", + "metadata": {}, + "source": [ + "## Construct a model\n", + "\n", + "This time we're going to use QKeras layers.\n", + "\n", + "**Note:** QKeras (https://github.com/google/qkeras) was originally developed for Keras v2. While the official version has not be an updated to Keras v3, the hls4ml community maintains a fork compatible with Keras v3: https://github.com/fastmachinelearning/qkerasV3. In this tutorial, we use the v3 version; though hls4ml still supports the v2 version of QKeras, with exactly the same syntax.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-5", + "metadata": {}, + "outputs": [], + "source": [ + "from keras.models import Sequential\n", + "from keras.optimizers import Adam\n", + "from keras.layers import Activation\n", + "from qkeras.qlayers import QDense, QActivation\n", + "from qkeras.quantizers import quantized_bits, quantized_relu" + ] + }, + { + "cell_type": "markdown", + "id": "cell-6", + "metadata": {}, + "source": [ + "We use `QDense` instead of `Dense`, and `QActivation` instead of `Activation`.\n", + "We specify `kernel_quantizer = quantized_bits(6, 0, alpha=1)`, which uses 6 bits (of which 0 are integer bits) for the weights and biases, and `quantized_relu(6)` for 6-bit ReLU activations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-7", + "metadata": {}, + "outputs": [], + "source": [ + "model = Sequential()\n", + "model.add(\n", + " QDense(\n", + " 64,\n", + " input_shape=(16,),\n", + " name='fc1',\n", + " kernel_quantizer=quantized_bits(6, 0, alpha=1),\n", + " bias_quantizer=quantized_bits(6, 0, alpha=1),\n", + " )\n", + ")\n", + "model.add(QActivation(activation=quantized_relu(6), name='relu1'))\n", + "model.add(\n", + " QDense(\n", + " 32,\n", + " name='fc2',\n", + " kernel_quantizer=quantized_bits(6, 0, alpha=1),\n", + " bias_quantizer=quantized_bits(6, 0, alpha=1),\n", + " )\n", + ")\n", + "model.add(QActivation(activation=quantized_relu(6), name='relu2'))\n", + "model.add(\n", + " QDense(\n", + " 32,\n", + " name='fc3',\n", + " kernel_quantizer=quantized_bits(6, 0, alpha=1),\n", + " bias_quantizer=quantized_bits(6, 0, alpha=1),\n", + " )\n", + ")\n", + "model.add(QActivation(activation=quantized_relu(6), name='relu3'))\n", + "model.add(\n", + " QDense(\n", + " 5,\n", + " name='output',\n", + " kernel_quantizer=quantized_bits(6, 0, alpha=1),\n", + " bias_quantizer=quantized_bits(6, 0, alpha=1),\n", + " )\n", + ")\n", + "model.add(Activation(activation='softmax', name='softmax'))\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "id": "cell-8", + "metadata": {}, + "source": [ + "## Train the model\n", + "\n", + "We use the Adam optimiser with categorical crossentropy loss.\n", + "The model isn't very complex, so this should take just a few minutes even on a CPU." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-9", + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(optimizer=Adam(learning_rate=1e-4), loss='categorical_crossentropy', metrics=['accuracy'])\n", + "model.fit(\n", + " X_train_val,\n", + " y_train_val,\n", + " batch_size=1024,\n", + " epochs=30,\n", + " validation_split=0.25,\n", + " shuffle=True,\n", + ")\n", + "os.makedirs('../models', exist_ok=True)\n", + "model.save('../models/qkeras_model_part2.h5')" + ] + }, + { + "cell_type": "markdown", + "id": "cell-10", + "metadata": {}, + "source": [ + "## Check performance\n", + "\n", + "How does the quantized model compare against the baseline from Part 1? Let's report the accuracy and make a ROC curve.\n", + "The baseline is shown with solid lines, the quantized model with dashed lines.\n", + "\n", + "We should also check that hls4ml can respect the choice to use 6-bits throughout the model and match the accuracy. We'll generate a configuration from this quantized model and plot its performance as the dotted line.\n", + "The generated configuration is printed out. You'll notice that it uses 7 bits for the type, but we specified 6 — that's because QKeras doesn't count the sign bit, so the type that actually gets used needs 1 more.\n", + "\n", + "We also use the `OutputRoundingSaturationMode` optimizer pass of hls4ml to set the Activation layers to round rather than truncate the cast. This is important for getting good model accuracy at small bit precision. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-11", + "metadata": {}, + "outputs": [], + "source": [ + "import hls4ml\n", + "\n", + "config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend='Vitis')\n", + "config['LayerName']['softmax']['exp_table_t'] = 'ap_fixed<18,8>'\n", + "config['LayerName']['softmax']['inv_table_t'] = 'ap_fixed<18,4>'\n", + "print('-----------------------------------')\n", + "plotting.print_dict(config)\n", + "print('-----------------------------------')\n", + "hls_model = hls4ml.converters.convert_from_keras_model(\n", + " model,\n", + " hls_config=config,\n", + " backend='Vitis',\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_qkeras_part2',\n", + " part='xcu200-fsgd2104-2-e',\n", + ")\n", + "hls_model.compile()\n", + "\n", + "y_qkeras = model.predict(np.ascontiguousarray(X_test))\n", + "y_hls = hls_model.predict(np.ascontiguousarray(X_test))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-12", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "from keras.models import load_model\n", + "\n", + "model_ref = load_model('../models/keras_model_part1.h5')\n", + "y_ref = model_ref.predict(np.ascontiguousarray(X_test))\n", + "\n", + "print('Accuracy baseline: {}'.format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_ref, axis=1))))\n", + "print('Accuracy quantized: {}'.format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_qkeras, axis=1))))\n", + "print('Accuracy hls4ml: {}'.format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_hls, axis=1))))\n", + "\n", + "fig, ax = plt.subplots(figsize=(9, 9))\n", + "_ = plotting.makeRoc(y_test, y_ref, list(classes))\n", + "plt.gca().set_prop_cycle(None)\n", + "_ = plotting.makeRoc(y_test, y_qkeras, list(classes), linestyle='--')\n", + "plt.gca().set_prop_cycle(None)\n", + "_ = plotting.makeRoc(y_test, y_hls, list(classes), linestyle=':')\n", + "\n", + "from matplotlib.lines import Line2D\n", + "from matplotlib.legend import Legend\n", + "\n", + "lines = [Line2D([0], [0], ls='-'), Line2D([0], [0], ls='--'), Line2D([0], [0], ls=':')]\n", + "leg = Legend(ax, lines, labels=['baseline', 'quantized', 'hls4ml'], loc='lower right', frameon=False)\n", + "ax.add_artist(leg)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-13", + "metadata": {}, + "source": [ + "## Synthesize\n", + "\n", + "Now let's synthesize this quantized model.\n", + "\n", + "**This can take several minutes.**\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-14", + "metadata": {}, + "outputs": [], + "source": [ + "hls_model.build(csim=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-15", + "metadata": {}, + "outputs": [], + "source": [ + "hls4ml.report.read_vivado_report('../hls4ml_prjs/hls4ml_prj_quantized_part2')" + ] + }, + { + "cell_type": "markdown", + "id": "aba31167", + "metadata": {}, + "source": [ + "Compare the DSP count above against the Part 1c baseline. With 6-bit quantization, every multiplication in this network is narrower than the ~10-bit threshold below which Vivado maps multiplications to LUT logic rather than DSP slices, significantly reducing the DSP usage." + ] + }, + { + "cell_type": "markdown", + "id": "cell-16", + "metadata": {}, + "source": [ + "## A note on HLS synthesis resource estimates\n", + "\n", + "\n", + "The resource numbers reported by Vitis HLS after HLS C-synthesis are **estimates** derived from the HLS's internal model and often do not truly reflect the final resource consumption of the model. These estimates **often overestimate** LUT consumption, sometimes by an order of magnitude.\n", + "\n", + "For a more accurate picture of resource consumption, you should run **Vivado synthesis** (`vsynth`). This invokes the full Vivado synthesis flow on the generated RTL, producing estimates that are much closer to what you would see after implementation (place-and-route).\n", + "\n", + "**This step can take 10–20 minutes.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-17", + "metadata": {}, + "outputs": [], + "source": [ + "hls_model.build(reset=False, csim=False, vsynth=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-18", + "metadata": {}, + "outputs": [], + "source": [ + "hls4ml.report.read_vivado_report('../hls4ml_prjs/hls4ml_prj_quantized_part2')" + ] + }, + { + "cell_type": "markdown", + "id": "14989120", + "metadata": {}, + "source": [ + "## Further reading\n", + "\n", + "For more details, see:\n", + "\n", + "- Coelho Jr., Kuusela, Zhuang et al., \"Ultra Low-latency, Low-area Inference Accelerators using Heterogeneous Deep Quantization with QKeras and hls4ml\", arXiv (2020), [arXiv:2006.10159](http://arxiv.org/abs/2006.10159)\n", + "- Coelho Jr., Kuusela, Li et al., \"Automatic heterogeneous quantization of deep neural networks for low-latency inference on the edge for particle detectors\", Nature Machine Intelligence (2021), [doi:10.1038/s42256-021-00356-5](https://www.nature.com/articles/s42256-021-00356-5)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "hls4ml-tutorial", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/2_quantization/2b_brevitas.ipynb b/2_quantization/2b_brevitas.ipynb new file mode 100644 index 00000000..00a61495 --- /dev/null +++ b/2_quantization/2b_brevitas.ipynb @@ -0,0 +1,430 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# Part 2b: Quantization with Brevitas\n", + "\n", + "In this notebook we train the same jet tagger architecture using [**Brevitas**](https://github.com/xilinx/brevitas) — AMD's/Xilinx's PyTorch-native quantization-aware training library — and convert it to an FPGA design via the **QONNX** (Quantized ONNX) frontend of hls4ml.\n", + "\n", + "The workflow in this notebook is:\n", + "1. Define a Brevitas model with `QuantLinear` and `QuantReLU` layers and train it\n", + "2. Export to QONNX with `export_qonnx`\n", + "3. Clean up the QONNX graph and import it into hls4ml via `convert_from_onnx_model`\n", + "\n", + "Make sure you have run `1_getting_started/1b_train_pytorch.ipynb` first so that the data and baseline PyTorch model are available." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import sys\n", + "\n", + "sys.path.append('..')\n", + "import plotting\n", + "\n", + "%matplotlib inline\n", + "seed = 0\n", + "np.random.seed(seed)\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "from torch.utils.data import DataLoader, TensorDataset\n", + "import brevitas.nn as qnn\n", + "from brevitas.export import export_qonnx\n", + "\n", + "torch.manual_seed(seed)\n", + "\n", + "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']" + ] + }, + { + "cell_type": "markdown", + "id": "cell-2", + "metadata": {}, + "source": [ + "## Load the jet tagging dataset\n", + "\n", + "We load the preprocessed arrays saved by `1b_train_pytorch.ipynb`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-3", + "metadata": {}, + "outputs": [], + "source": [ + "X_train_val = np.load('../data/jet-tagging/X_train_val.npy')\n", + "X_test = np.load('../data/jet-tagging/X_test.npy')\n", + "y_train_val = np.load('../data/jet-tagging/y_train_val.npy')\n", + "y_test = np.load('../data/jet-tagging/y_test.npy')\n", + "classes = np.load('../data/jet-tagging/classes.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-4", + "metadata": {}, + "source": [ + "## Define a Brevitas model\n", + "\n", + "Brevitas replaces standard PyTorch layers with quantized equivalents:\n", + "- `QuantLinear` — linear layer with quantized weights (and optionally biases)\n", + "- `QuantReLU` — ReLU that clips and quantizes activations to a fixed bit-width\n", + "\n", + "We use `weight_bit_width=6` and `bit_width=6` throughout, matching the 6-bit precision from Part 2a.\n", + "Unlike regular PyTorch, every forward pass applies the quantization, so the model trains with the actual fixed-point precisions used on the FPGA." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-5", + "metadata": {}, + "outputs": [], + "source": [ + "class JetTaggerBrevitas(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.fc1 = qnn.QuantLinear(16, 64, bias=False, weight_bit_width=6)\n", + " self.relu1 = qnn.QuantReLU(bit_width=6)\n", + " self.fc2 = qnn.QuantLinear(64, 32, bias=False, weight_bit_width=6)\n", + " self.relu2 = qnn.QuantReLU(bit_width=6)\n", + " self.fc3 = qnn.QuantLinear(32, 32, bias=False, weight_bit_width=6)\n", + " self.relu3 = qnn.QuantReLU(bit_width=6)\n", + " self.output = qnn.QuantLinear(32, 5, bias=False, weight_bit_width=6)\n", + "\n", + " def forward(self, x):\n", + " x = self.relu1(self.fc1(x))\n", + " x = self.relu2(self.fc2(x))\n", + " x = self.relu3(self.fc3(x))\n", + " x = self.output(x)\n", + " return torch.softmax(x, dim=1)\n", + "\n", + "\n", + "model = JetTaggerBrevitas()\n", + "print(model)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-6", + "metadata": {}, + "source": [ + "## Train the model\n", + "\n", + "Training is identical to standard PyTorch. Because softmax is the final operation, we use `NLLLoss` on the log of the model output." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-7", + "metadata": {}, + "outputs": [], + "source": [ + "n_train = int(len(X_train_val) * 0.75)\n", + "\n", + "X_tr = torch.FloatTensor(X_train_val[:n_train])\n", + "y_tr = torch.LongTensor(np.argmax(y_train_val[:n_train], axis=1))\n", + "X_val = torch.FloatTensor(X_train_val[n_train:])\n", + "y_val = torch.LongTensor(np.argmax(y_train_val[n_train:], axis=1))\n", + "\n", + "loader = DataLoader(TensorDataset(X_tr, y_tr), batch_size=1024, shuffle=True)\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n", + "criterion = nn.NLLLoss()\n", + "\n", + "for epoch in range(20):\n", + " model.train()\n", + " for X_batch, y_batch in loader:\n", + " optimizer.zero_grad()\n", + " loss = criterion(torch.log(model(X_batch).clamp(min=1e-7)), y_batch)\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " model.eval()\n", + " with torch.no_grad():\n", + " val_loss = criterion(torch.log(model(X_val).clamp(min=1e-7)), y_val).item()\n", + " print(f'Epoch {epoch + 1:2d} val_loss={val_loss:.4f}')" + ] + }, + { + "cell_type": "markdown", + "id": "cell-8", + "metadata": {}, + "source": [ + "## Export to QONNX\n", + "\n", + "Brevitas provides the `export_qonnx` function to serialize the trained model to an ONNX file that preserves the quantization (Quant nodes). This QONNX format is what hls4ml reads to infer per-layer precision.\n", + "\n", + "After exporting, we run two cleanup steps:\n", + "1. `qonnx.util.cleanup` — removes redundant nodes and canonicalises the graph\n", + "2. `GemmToMatMul` — PyTorch exports linear layers as `Gemm` ops; hls4ml expects `MatMul`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-9", + "metadata": {}, + "outputs": [], + "source": [ + "import qonnx.util.cleanup\n", + "from qonnx.core.modelwrapper import ModelWrapper\n", + "from qonnx.transformation.gemm_to_matmul import GemmToMatMul\n", + "\n", + "os.makedirs('../models', exist_ok=True)\n", + "raw_path = '../models/brevitas_model_part2b_raw.onnx'\n", + "final_path = '../models/brevitas_model_part2b.onnx'\n", + "\n", + "model.eval()\n", + "export_qonnx(model, args=torch.randn(1, 16), export_path=raw_path, opset_version=14, dynamo=False)\n", + "print(f'Exported raw QONNX to {raw_path}')\n", + "\n", + "# Clean and convert Gemm → MatMul\n", + "qonnx.util.cleanup.cleanup(raw_path, out_file=final_path)\n", + "model_wrapper = ModelWrapper(final_path)\n", + "model_wrapper = model_wrapper.transform(GemmToMatMul())\n", + "model_wrapper = qonnx.util.cleanup.cleanup_model(model_wrapper)\n", + "model_wrapper.save(final_path)\n", + "print(f'Cleaned QONNX saved to {final_path}')" + ] + }, + { + "cell_type": "markdown", + "id": "cell-10", + "metadata": {}, + "source": [ + "## Check performance (float)\n", + "\n", + "Before converting to HLS, let's check the trained model's accuracy in float simulation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-11", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "\n", + "model.eval()\n", + "with torch.no_grad():\n", + " y_brevitas = model(torch.FloatTensor(X_test)).numpy()\n", + "\n", + "print('Accuracy (Brevitas float): {}'.format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_brevitas, axis=1))))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-12", + "metadata": {}, + "source": [ + "## Convert to hls4ml via QONNX\n", + "\n", + "`config_from_onnx_model` reads the Quant nodes embedded in the QONNX graph and derives per-layer precision automatically. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-13", + "metadata": {}, + "outputs": [], + "source": [ + "import hls4ml\n", + "\n", + "config = hls4ml.utils.config_from_onnx_model(\n", + " model_wrapper,\n", + " granularity='name',\n", + " backend='Vitis',\n", + ")\n", + "\n", + "for layer in config.get('LayerName', {}):\n", + " if layer.startswith('Softmax'):\n", + " config['LayerName'][layer]['Implementation'] = 'legacy'\n", + "\n", + "print('-----------------------------------')\n", + "plotting.print_dict(config)\n", + "print('-----------------------------------')\n", + "\n", + "hls_model = hls4ml.converters.convert_from_onnx_model(\n", + " model_wrapper,\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_brevitas_part2b',\n", + " backend='Vitis',\n", + " hls_config=config,\n", + " part='xcu200-fsgd2104-2-e',\n", + ")\n", + "hls_model.compile()\n", + "y_hls = hls_model.predict(np.ascontiguousarray(X_test, dtype=np.float32))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-14", + "metadata": {}, + "source": [ + "## Compare\n", + "\n", + "We compare the baseline PyTorch model (Part 1b), the Brevitas quantized model, and the hls4ml emulation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-15", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "\n", + "\n", + "# Load the baseline PyTorch model from Part 1b\n", + "class JetTagger(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(16, 64)\n", + " self.fc2 = nn.Linear(64, 32)\n", + " self.fc3 = nn.Linear(32, 32)\n", + " self.output = nn.Linear(32, 5)\n", + "\n", + " def forward(self, x):\n", + " x = torch.relu(self.fc1(x))\n", + " x = torch.relu(self.fc2(x))\n", + " x = torch.relu(self.fc3(x))\n", + " return torch.softmax(self.output(x), dim=1)\n", + "\n", + "\n", + "model_ref = JetTagger()\n", + "model_ref.load_state_dict(torch.load('../models/pytorch_weights_part1.pt'))\n", + "model_ref.eval()\n", + "with torch.no_grad():\n", + " y_ref = model_ref(torch.FloatTensor(X_test)).numpy()\n", + "\n", + "print('Accuracy baseline (PyTorch): {}'.format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_ref, axis=1))))\n", + "print('Accuracy Brevitas (float): {}'.format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_brevitas, axis=1))))\n", + "print('Accuracy hls4ml: {}'.format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_hls, axis=1))))\n", + "\n", + "fig, ax = plt.subplots(figsize=(9, 9))\n", + "_ = plotting.makeRoc(y_test, y_ref, list(classes))\n", + "plt.gca().set_prop_cycle(None)\n", + "_ = plotting.makeRoc(y_test, y_brevitas, list(classes), linestyle='--')\n", + "plt.gca().set_prop_cycle(None)\n", + "_ = plotting.makeRoc(y_test, y_hls, list(classes), linestyle=':')\n", + "\n", + "from matplotlib.lines import Line2D\n", + "from matplotlib.legend import Legend\n", + "\n", + "lines = [Line2D([0], [0], ls='-'), Line2D([0], [0], ls='--'), Line2D([0], [0], ls=':')]\n", + "leg = Legend(ax, lines, labels=['baseline (PyTorch)', 'Brevitas (float)', 'hls4ml'], loc='lower right', frameon=False)\n", + "ax.add_artist(leg)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-16", + "metadata": {}, + "source": [ + "## Synthesize\n", + "\n", + "Now let's run Vitis HLS C-synthesis to get a first estimate of latency and resource usage.\n", + "\n", + "**This can take several minutes.**\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-17", + "metadata": {}, + "outputs": [], + "source": [ + "hls_model.build(csim=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-18", + "metadata": {}, + "outputs": [], + "source": [ + "hls4ml.report.read_vivado_report('../hls4ml_prjs/hls4ml_prj_brevitas_part2b')" + ] + }, + { + "cell_type": "markdown", + "id": "5fa51e96", + "metadata": {}, + "source": [ + "Compare the DSP count above against the Part 1c baseline. With 6-bit quantization, every multiplication in this network is narrower than the ~10-bit threshold below which Vivado maps multiplications to LUT logic rather than DSP slices, significantly reducing the DSP usage." + ] + }, + { + "cell_type": "markdown", + "id": "cell-19", + "metadata": {}, + "source": [ + "## A note on HLS synthesis resource estimates\n", + "\n", + "The resource numbers reported by Vitis HLS after HLS C-synthesis are **estimates** derived from the HLS's internal model and often do not truly reflect the final resource consumption of the model. These estimates **often overestimate** LUT consumption, sometimes by an order of magnitude.\n", + "\n", + "For a more accurate picture of resource consumption, you should run **Vivado synthesis** (`vsynth`). This invokes the full Vivado synthesis flow on the generated RTL, producing estimates that are much closer to what you would see after implementation (place-and-route).\n", + "\n", + "**This step can take 10–20 minutes.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-20", + "metadata": {}, + "outputs": [], + "source": [ + "hls_model.build(reset=False, csim=False, vsynth=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-21", + "metadata": {}, + "outputs": [], + "source": [ + "hls4ml.report.read_vivado_report('../hls4ml_prjs/hls4ml_prj_brevitas_part2b')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "hls4ml-tutorial", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/3_advanced_config/3a_reuse_factor.ipynb b/3_advanced_config/3a_reuse_factor.ipynb new file mode 100644 index 00000000..ac457b65 --- /dev/null +++ b/3_advanced_config/3a_reuse_factor.ipynb @@ -0,0 +1,300 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# Part 3a: Reuse Factor\n", + "\n", + "In Part 1c we used the default hls4ml configuration, which maps the entire neural network onto the FPGA with maximum parallelism, so that all multiply-accumulate operation run in parallel. This gives the lowest possible latency, but consumes a lot of resources.\n", + "\n", + "The **reuse factor (RF)** is a variable in hls4ml for tuning the trade-off between resource consumption and latency. In this notebook we explore how changing it affects the synthesized design.\n", + "\n", + "Run either `1_getting_started/1a_train_keras.ipynb` or `1_getting_started/1b_train_pytorch.ipynb` first, then set `MODEL_TYPE` below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-1", + "metadata": {}, + "outputs": [], + "source": [ + "MODEL_TYPE = 'keras' # set to 'pytorch' if you used the PyTorch notebook in Part 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-2", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['KERAS_BACKEND'] = 'tensorflow'\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import sys\n", + "\n", + "sys.path.append('..')\n", + "import plotting\n", + "import hls4ml\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "%matplotlib inline\n", + "\n", + "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']\n", + "\n", + "# Load the data\n", + "X_test = np.ascontiguousarray(np.load('../data/jet-tagging/X_test.npy'), dtype=np.float32)\n", + "y_test = np.load('../data/jet-tagging/y_test.npy')\n", + "classes = np.load('../data/jet-tagging/classes.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-3", + "metadata": {}, + "source": [ + "## Load the trained model\n", + "\n", + "Load the model saved by Part 1a or 1b." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-4", + "metadata": {}, + "outputs": [], + "source": [ + "if MODEL_TYPE == 'keras':\n", + " import tensorflow as tf\n", + "\n", + " trained_model = tf.keras.models.load_model('../models/keras_model_part1.h5')\n", + " y_ref = trained_model.predict(X_test)\n", + "\n", + "elif MODEL_TYPE == 'pytorch':\n", + " import torch\n", + " import torch.nn as nn\n", + "\n", + " class JetTagger(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(16, 64)\n", + " self.fc2 = nn.Linear(64, 32)\n", + " self.fc3 = nn.Linear(32, 32)\n", + " self.output = nn.Linear(32, 5)\n", + "\n", + " def forward(self, x):\n", + " x = torch.relu(self.fc1(x))\n", + " x = torch.relu(self.fc2(x))\n", + " x = torch.relu(self.fc3(x))\n", + " return torch.softmax(self.output(x), dim=1)\n", + "\n", + " trained_model = JetTagger()\n", + " trained_model.load_state_dict(torch.load('../models/pytorch_weights_part1.pt'))\n", + " trained_model.eval()\n", + " with torch.no_grad():\n", + " y_ref = trained_model(torch.FloatTensor(X_test)).numpy()\n", + "\n", + "print('Accuracy: {:.4f}'.format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_ref, axis=1))))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-5", + "metadata": {}, + "source": [ + "## What is the ReuseFactor?\n", + "\n", + "In the default (`ReuseFactor = 1`) configuration, hls4ml instantiates one multiplier for every weight in the network. All multiplications for a given layer happen in a single clock cycle, giving the minimum possible latency — but using the most multipliers.\n", + "\n", + "Setting `ReuseFactor = N` tells hls4ml to time-multiplex the same multiplier hardware across `N` weight-input pairs. This means the layer takes `N` clock cycles to compute instead of one, but uses roughly `1/N` as many multipliers.\n", + "\n", + "![Reuse factor diagram](../images/part3a_reuse_factor.png)\n", + "\n", + "The reuse factor must evenly divide the number of weights in each layer. For example, the first layer has `16 × 64 = 1024` weights, so valid reuse factors include 1, 2, 4, 8, 16, 32, 64, etc.\n", + "\n", + "Changing the reuse factor does **not** change the model accuracy — the same arithmetic is performed, just spread over more clock cycles. We will verify this below." + ] + }, + { + "cell_type": "markdown", + "id": "cell-6", + "metadata": {}, + "source": [ + "## Set ReuseFactor = 4\n", + "\n", + "Let's create a new configuration with `ReuseFactor = 4` set globally. Note that we use `granularity='model'` here, which applies one set of defaults to all layers — equivalent to the config from Part 1c." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-7", + "metadata": {}, + "outputs": [], + "source": [ + "if MODEL_TYPE == 'keras':\n", + " config = hls4ml.utils.config_from_keras_model(trained_model, granularity='model', backend='Vitis')\n", + "elif MODEL_TYPE == 'pytorch':\n", + " config = hls4ml.utils.config_from_pytorch_model(trained_model, input_shape=(16,), granularity='model', backend='Vitis')\n", + "\n", + "config['Model']['ReuseFactor'] = 4\n", + "print('-----------------------------------')\n", + "plotting.print_dict(config)\n", + "print('-----------------------------------')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-8", + "metadata": {}, + "outputs": [], + "source": [ + "if MODEL_TYPE == 'keras':\n", + " hls_model = hls4ml.converters.convert_from_keras_model(\n", + " trained_model,\n", + " hls_config=config,\n", + " backend='Vitis',\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_reuse_part3a',\n", + " part='xcu200-fsgd2104-2-e',\n", + " )\n", + "elif MODEL_TYPE == 'pytorch':\n", + " hls_model = hls4ml.converters.convert_from_pytorch_model(\n", + " trained_model,\n", + " hls_config=config,\n", + " backend='Vitis',\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_reuse_part3a',\n", + " part='xcu200-fsgd2104-2-e',\n", + " )\n", + "\n", + "hls_model.compile()\n", + "y_hls = hls_model.predict(X_test)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-9", + "metadata": {}, + "source": [ + "## Compare\n", + "\n", + "Changing the reuse factor only affects resource usage and latency — not accuracy. Let's verify that the accuracy and ROC curves are identical." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-10", + "metadata": {}, + "outputs": [], + "source": [ + "print('{} Accuracy: {:.4f}'.format(MODEL_TYPE, accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_ref, axis=1))))\n", + "print('hls4ml Accuracy: {:.4f}'.format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_hls, axis=1))))\n", + "\n", + "fig, ax = plt.subplots(figsize=(9, 9))\n", + "_ = plotting.makeRoc(y_test, y_ref, list(classes))\n", + "plt.gca().set_prop_cycle(None)\n", + "_ = plotting.makeRoc(y_test, y_hls, list(classes), linestyle='--')\n", + "\n", + "from matplotlib.lines import Line2D\n", + "from matplotlib.legend import Legend\n", + "\n", + "lines = [Line2D([0], [0], ls='-'), Line2D([0], [0], ls='--')]\n", + "leg = Legend(ax, lines, labels=[MODEL_TYPE, 'hls4ml (RF=4)'], loc='lower right', frameon=False)\n", + "ax.add_artist(leg)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-11", + "metadata": {}, + "source": [ + "## Synthesize\n", + "\n", + "Now let's synthesize with `ReuseFactor = 4` and compare the resource report against the Part 1c baseline (`ReuseFactor = 1`) to see the effect.\n", + "\n", + "**This can take several minutes.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-12", + "metadata": {}, + "outputs": [], + "source": [ + "hls_model.build(csim=False)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-13", + "metadata": {}, + "source": [ + "## Compare reports\n", + "\n", + "Print both reports and compare the DSP and LUT usage. With `ReuseFactor = 4`, you should see roughly a quarter as many DSPs as in the Part 1c baseline, at the cost of approximately four times the latency (in clock cycles)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-14", + "metadata": {}, + "outputs": [], + "source": [ + "print('ReuseFactor = 4:')\n", + "hls4ml.report.read_vivado_report('../hls4ml_prjs/hls4ml_prj_reuse_part3a')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-15", + "metadata": {}, + "outputs": [], + "source": [ + "print('ReuseFactor = 1 (Part 1c baseline):')\n", + "hls4ml.report.read_vivado_report('../hls4ml_prjs/hls4ml_prj_base_part1')" + ] + }, + { + "cell_type": "markdown", + "id": "63bbdc7c", + "metadata": {}, + "source": [ + "## Further reading\n", + "\n", + "For more details, see: Schulte, Ramhorst, Sun et al., \"hls4ml: A Flexible, Open-Source Platform for Deep Learning Acceleration on Reconfigurable Hardware\", ACM Trans. Reconfigurable Technol. Syst. (2026), [doi:10.1145/3801979](https://dl.acm.org/doi/abs/10.1145/3801979)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "hls4ml-tutorial", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/3_advanced_config/3b_profiling.ipynb b/3_advanced_config/3b_profiling.ipynb new file mode 100644 index 00000000..d3e21c5a --- /dev/null +++ b/3_advanced_config/3b_profiling.ipynb @@ -0,0 +1,424 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# Part 3b: Automatic precision inference, profiling & tracing\n", + "\n", + "In this notebook, we will explore:\n", + " \n", + "- **Automatic precision inference**, which lets hls4ml automatically compute a conservative but layer-specific precision for each weight, bias, accumulator, and output tensor.\n", + " \n", + "- **Profiling**, to visualise the numerical range of the weights and activations\n", + " \n", + "- **Tracing**, to analyze per-layer outputs and verify accuracy is preserved.\n", + "\n", + "Run either `1_getting_started/1a_train_keras.ipynb` or `1_getting_started/1b_train_pytorch.ipynb` first, then set `MODEL_TYPE` below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-1", + "metadata": {}, + "outputs": [], + "source": [ + "MODEL_TYPE = 'keras' # set to 'pytorch' if you used the PyTorch notebook" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-2", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['KERAS_BACKEND'] = 'tensorflow'\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import sys\n", + "\n", + "sys.path.append('..')\n", + "import plotting\n", + "import hls4ml\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']\n", + "\n", + "%matplotlib inline\n", + "\n", + "X_test = np.ascontiguousarray(np.load('../data/jet-tagging/X_test.npy'), dtype=np.float32)\n", + "y_test = np.load('../data/jet-tagging/y_test.npy')\n", + "classes = np.load('../data/jet-tagging/classes.npy', allow_pickle=True)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-3", + "metadata": {}, + "source": [ + "## Load the trained model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-4", + "metadata": {}, + "outputs": [], + "source": [ + "if MODEL_TYPE == 'keras':\n", + " import tensorflow as tf\n", + "\n", + " trained_model = tf.keras.models.load_model('../models/keras_model_part1.h5')\n", + " y_ref = trained_model.predict(X_test)\n", + "\n", + "elif MODEL_TYPE == 'pytorch':\n", + " import torch\n", + " import torch.nn as nn\n", + "\n", + " class JetTagger(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(16, 64)\n", + " self.fc2 = nn.Linear(64, 32)\n", + " self.fc3 = nn.Linear(32, 32)\n", + " self.output = nn.Linear(32, 5)\n", + "\n", + " def forward(self, x):\n", + " x = torch.relu(self.fc1(x))\n", + " x = torch.relu(self.fc2(x))\n", + " x = torch.relu(self.fc3(x))\n", + " return torch.softmax(self.output(x), dim=1)\n", + "\n", + " trained_model = JetTagger()\n", + " trained_model.load_state_dict(torch.load('../models/pytorch_weights_part1.pt'))\n", + " trained_model.eval()\n", + " with torch.no_grad():\n", + " y_ref = trained_model(torch.FloatTensor(X_test)).numpy()\n", + "\n", + "print('Accuracy: {:.4f}'.format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_ref, axis=1))))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-5", + "metadata": {}, + "source": [ + "## Make an hls4ml config with `granularity='name'`\n", + "\n", + "In Part 1c we used `granularity='model'`, which applies the same configuration to every layer. Setting `granularity='name'` instead gives each layer its own config sub-dictionary, allowing us to tune the reuse factor, strategy, precision etc. per layer. \n", + "\n", + "When using `granularity='name'`, automatic precision inference is automatically enabled. The `auto` precision is computed conservatively: it guarantees no overflow or truncation based purely on the input bit-widths, without looking at the actual data distribution.\n", + "\n", + "For example, the entry for `fc1` will look like:\n", + "```\n", + "LayerName:\n", + " fc1:\n", + " Trace: False\n", + " Precision:\n", + " weight: auto\n", + " bias: auto\n", + " result: auto\n", + " accum: auto\n", + " ReuseFactor: 1\n", + "```\n", + "\n", + "With `auto` precision:\n", + "- `weight` and `bias` default to the global model precision (`ap_fixed<16,6>`).\n", + "- `result` and `accum` are computed to be wide enough to hold the worst-case accumulation without overflow — these are often much wider than 16 bits. While ensuring no accuracy loss, it may come at the expense of resources.\n", + "\n", + "This configuration is a useful starting point for manual tuning -- you can inspect the profiling plots below to decide which layers can safely use narrower types and accordingly update the config." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-6", + "metadata": {}, + "outputs": [], + "source": [ + "if MODEL_TYPE == 'keras':\n", + " config = hls4ml.utils.config_from_keras_model(trained_model, granularity='name', backend='Vitis')\n", + "elif MODEL_TYPE == 'pytorch':\n", + " config = hls4ml.utils.config_from_pytorch_model(trained_model, input_shape=(16,), granularity='name', backend='Vitis')\n", + "\n", + "print('-----------------------------------')\n", + "plotting.print_dict(config)\n", + "print('-----------------------------------')" + ] + }, + { + "cell_type": "markdown", + "id": "cell-7", + "metadata": {}, + "source": [ + "## Profiling\n", + "\n", + "The `hls4ml.model.profiling.numerical` function plots the distribution of weights and biases as a box-and-whisker chart. The **grey boxes** show the range representable with the data types set in the hls4ml config.\n", + "\n", + "The rule of thumb:\n", + "- The grey box should cover the full whisker **to the right** (large values) — otherwise weights saturate or wrap around.\n", + "- It is acceptable for the box not to reach the left whisker (small values): those weights are simply rounded to zero, which is *often* harmless.\n", + "\n", + "Providing data (here the first 1000 test samples for speed) also shows the same distributions at the **output of each layer**, which reveals whether the activation dynamic range is well-matched to the fixed-point type.\n", + "\n", + "**Note:** if you find that the default 16-bit precision is too conservative for your model, the best solution is quantization-aware training (QAT) (see Part 2). Post-training, manual tuning (this notebook) is a useful alternative when retraining is not an option, but it requires more iterations and careful precision tuning." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-8", + "metadata": {}, + "outputs": [], + "source": [ + "from hls4ml.model.profiling import numerical\n", + "\n", + "for layer in config['LayerName'].keys():\n", + " config['LayerName'][layer]['Trace'] = True\n", + "\n", + "if MODEL_TYPE == 'keras':\n", + " hls_model = hls4ml.converters.convert_from_keras_model(\n", + " trained_model,\n", + " hls_config=config,\n", + " backend='Vitis',\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_profiling_part3b',\n", + " part='xcu200-fsgd2104-2-e',\n", + " )\n", + "elif MODEL_TYPE == 'pytorch':\n", + " hls_model = hls4ml.converters.convert_from_pytorch_model(\n", + " trained_model,\n", + " hls_config=config,\n", + " backend='Vitis',\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_profiling_part3b',\n", + " part='xcu200-fsgd2104-2-e',\n", + " )\n", + "\n", + "numerical(model=trained_model, hls_model=hls_model, X=X_test[:1000])" + ] + }, + { + "cell_type": "markdown", + "id": "cell-9", + "metadata": {}, + "source": [ + "## Customise precision\n", + "\n", + "After inspecting the profiling plot, let's try narrowing the weight precision of `fc1` from 16 bits to 8 bits (`ap_fixed<8,2>` — 8 total bits, 2 integer bits). This reduces the multiplier width and can save significant LUT and DSP resources.\n", + "\n", + "**Note on the output layer:** Using `auto` precision can produce an accumulator at the output of the last fully-connected layer that is wider than the softmax look-up tables can handle. We therefore manually cap it with `fixed<16,6,RND,SAT>`, which also enables rounding and saturation — important when narrowing any type that feeds into a non-linear function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-10", + "metadata": {}, + "outputs": [], + "source": [ + "config['LayerName']['fc1']['Precision']['weight'] = 'ap_fixed<8,2>'\n", + "config['LayerName']['output']['Precision']['result'] = 'fixed<16,6,RND,SAT>'\n", + "\n", + "if MODEL_TYPE == 'keras':\n", + " hls_model = hls4ml.converters.convert_from_keras_model(\n", + " trained_model,\n", + " hls_config=config,\n", + " backend='Vitis',\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_profiling_part3b',\n", + " part='xcu200-fsgd2104-2-e',\n", + " )\n", + "elif MODEL_TYPE == 'pytorch':\n", + " hls_model = hls4ml.converters.convert_from_pytorch_model(\n", + " trained_model,\n", + " hls_config=config,\n", + " backend='Vitis',\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_profiling_part3b',\n", + " part='xcu200-fsgd2104-2-e',\n", + " )\n", + "\n", + "# Plot weight-only profile (no data) to see the updated precision boxes\n", + "numerical(model=trained_model, hls_model=hls_model)\n", + "hls4ml.utils.plot_model(hls_model, show_shapes=True, show_precision=True, to_file=None)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-11", + "metadata": {}, + "source": [ + "## Trace\n", + "\n", + "When using customised per-layer precision, it is useful to collect the intermediate output of each layer and compare it between the original model and the hls4ml emulation. We enable this by setting `Trace = True` for every layer in the config, which we already did above.\n", + "\n", + "Re-convert with all traces enabled so the compiled model will record layer outputs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-12", + "metadata": {}, + "outputs": [], + "source": [ + "for layer in config['LayerName'].keys():\n", + " config['LayerName'][layer]['Trace'] = True\n", + "\n", + "if MODEL_TYPE == 'keras':\n", + " hls_model = hls4ml.converters.convert_from_keras_model(\n", + " trained_model,\n", + " hls_config=config,\n", + " backend='Vitis',\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_profiling_part3b',\n", + " part='xcu200-fsgd2104-2-e',\n", + " )\n", + "elif MODEL_TYPE == 'pytorch':\n", + " hls_model = hls4ml.converters.convert_from_pytorch_model(\n", + " trained_model,\n", + " hls_config=config,\n", + " backend='Vitis',\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_profiling_part3b',\n", + " part='xcu200-fsgd2104-2-e',\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "cell-13", + "metadata": {}, + "source": [ + "## Compile, trace, predict\n", + "\n", + "Compile the hls4ml model and call `hls_model.trace` instead of `hls_model.predict`. This returns both the final predictions **and** a dictionary of intermediate layer outputs — one array per layer, keyed by layer name.\n", + "\n", + "We collect the same dictionary from the original model for comparison. We only trace the first 1000 samples since tracing is slower than a plain forward pass." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-14", + "metadata": {}, + "outputs": [], + "source": [ + "hls_model.compile()\n", + "hls4ml_pred, hls4ml_trace = hls_model.trace(X_test[:1000])\n", + "y_hls = hls_model.predict(X_test)\n", + "\n", + "if MODEL_TYPE == 'keras':\n", + " from hls4ml.model.profiling import get_ymodel_keras\n", + "\n", + " ref_trace = get_ymodel_keras(trained_model, X_test[:1000])\n", + "\n", + "elif MODEL_TYPE == 'pytorch':\n", + " ref_trace = {}\n", + " hooks = []\n", + "\n", + " def make_hook(name):\n", + " def hook(module, inp, out):\n", + " if isinstance(out, torch.Tensor):\n", + " ref_trace[name] = out.detach().numpy()\n", + "\n", + " return hook\n", + "\n", + " for name, module in trained_model.named_modules():\n", + " if name:\n", + " hooks.append(module.register_forward_hook(make_hook(name)))\n", + "\n", + " trained_model.eval()\n", + " with torch.no_grad():\n", + " trained_model(torch.FloatTensor(X_test[:1000]))\n", + "\n", + " for h in hooks:\n", + " h.remove()" + ] + }, + { + "cell_type": "markdown", + "id": "cell-15", + "metadata": {}, + "source": [ + "## Inspect\n", + "\n", + "We can now print, plot, or otherwise compare the output of each layer between the original model and the hls4ml fixed-point emulation. This makes it easy to spot which layer first deviates — a sign that the precision there is too narrow.\n", + "\n", + "Let's print the first-layer output for the very first test sample." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-16", + "metadata": {}, + "outputs": [], + "source": [ + "layer_name = 'fc1'\n", + "print(f\"Original model layer '{layer_name}', first sample:\")\n", + "print(ref_trace[layer_name][0])\n", + "print(f\"hls4ml layer '{layer_name}', first sample:\")\n", + "print(hls4ml_trace[layer_name][0])" + ] + }, + { + "cell_type": "markdown", + "id": "cell-17", + "metadata": {}, + "source": [ + "## Compare\n", + "\n", + "Let's see whether the 8-bit weight precision on `fc1` has degraded overall accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-18", + "metadata": {}, + "outputs": [], + "source": [ + "print('{} Accuracy: {:.4f}'.format(MODEL_TYPE, accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_ref, axis=1))))\n", + "print('hls4ml Accuracy: {:.4f}'.format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_hls, axis=1))))\n", + "\n", + "fig, ax = plt.subplots(figsize=(9, 9))\n", + "_ = plotting.makeRoc(y_test, y_ref, list(classes))\n", + "plt.gca().set_prop_cycle(None)\n", + "_ = plotting.makeRoc(y_test, y_hls, list(classes), linestyle='--')\n", + "\n", + "from matplotlib.lines import Line2D\n", + "from matplotlib.legend import Legend\n", + "\n", + "lines = [Line2D([0], [0], ls='-'), Line2D([0], [0], ls='--')]\n", + "leg = Legend(ax, lines, labels=[MODEL_TYPE, 'hls4ml (8-bit fc1)'], loc='lower right', frameon=False)\n", + "ax.add_artist(leg)" + ] + }, + { + "cell_type": "markdown", + "id": "6afa959e", + "metadata": {}, + "source": [ + "## Further reading\n", + "\n", + "For more details, see: Schulte, Ramhorst, Sun et al., \"hls4ml: A Flexible, Open-Source Platform for Deep Learning Acceleration on Reconfigurable Hardware\", ACM Trans. Reconfigurable Technol. Syst. (2026), [doi:10.1145/3801979](https://dl.acm.org/doi/abs/10.1145/3801979)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/4_advanced_models/4a_qkeras_cnn_svhn.ipynb b/4_advanced_models/4a_qkeras_cnn_svhn.ipynb new file mode 100644 index 00000000..8d4068f8 --- /dev/null +++ b/4_advanced_models/4a_qkeras_cnn_svhn.ipynb @@ -0,0 +1,459 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4a-0", + "metadata": {}, + "source": [ + "# Part 4a: Convolutional Neural Networks with QKeras on the SVHN dataset\n", + "\n", + "In this notebook we train a quantized convolutional neural network (CNN) on the [Street View House Numbers (SVHN)](http://ufldl.stanford.edu/housenumbers/) dataset and deploy it with hls4ml.\n", + "\n", + "The SVHN dataset consists of real-world images of house numbers extracted from Google Street View, cropped to 32×32 RGB pixels. Unlike MNIST it is a harder, more realistic problem: images can contain more than one digit, and the centre digit defines the label. Each image belongs to one of 10 classes (digits 0–9).\n", + "\n", + "![SVHN examples from the test set](../images/part4a_test_images.png)\n", + "\n", + "The dataset has 73,257 training images and 26,032 test images." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a-1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['KERAS_BACKEND'] = 'tensorflow'\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import scipy.io\n", + "import time\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "sys.path.append('..')\n", + "import plotting\n", + "\n", + "import tensorflow as tf\n", + "import keras\n", + "from keras.layers import Input, MaxPooling2D, Activation, Flatten, Dense, BatchNormalization\n", + "from keras.models import Model\n", + "\n", + "from qkeras import QActivation, QDense, QConv2D\n", + "\n", + "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']" + ] + }, + { + "cell_type": "markdown", + "id": "4a-2", + "metadata": {}, + "source": [ + "## Fetch the SVHN dataset\n", + "\n", + "We download the SVHN `.mat` files directly from the [official source](http://ufldl.stanford.edu/housenumbers/) and load them with `scipy.io`. The first 75% of the training split is used for training and the remaining 25% for validation.\n", + "\n", + "To save time we do not use the `extra` split (531k additional images); adding it when loading is the most effective way to improve accuracy if you have time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a-3", + "metadata": {}, + "outputs": [], + "source": [ + "DATA_DIR = Path('../data/svhn')\n", + "DATA_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "for fname in ['train_32x32.mat', 'test_32x32.mat']:\n", + " if not (DATA_DIR / fname).exists():\n", + " keras.utils.get_file(\n", + " fname=fname,\n", + " origin=f'http://ufldl.stanford.edu/housenumbers/{fname}',\n", + " cache_dir=str(DATA_DIR.parent),\n", + " cache_subdir='svhn',\n", + " )\n", + "\n", + "\n", + "def load_mat(path):\n", + " data = scipy.io.loadmat(path)\n", + " X = data['X'].transpose(3, 0, 1, 2).astype(np.float32) / 255.0\n", + " y = data['y'].flatten() % 10 # SVHN labels are 1–10; % 10 maps 10 → 0\n", + " Y = np.eye(10, dtype=np.float32)[y]\n", + " return X, Y\n", + "\n", + "\n", + "X_all, Y_all = load_mat(DATA_DIR / 'train_32x32.mat')\n", + "X_test, Y_test = load_mat(DATA_DIR / 'test_32x32.mat')\n", + "\n", + "# Shuffle before splitting — the .mat file is ordered by class\n", + "rng = np.random.default_rng(42)\n", + "perm = rng.permutation(len(X_all))\n", + "X_all, Y_all = X_all[perm], Y_all[perm]\n", + "\n", + "val_idx = int(0.75 * len(X_all))\n", + "X_train, Y_train = X_all[:val_idx], Y_all[:val_idx]\n", + "X_val, Y_val = X_all[val_idx:], Y_all[val_idx:]\n", + "\n", + "input_shape = X_train.shape[1:] # (32, 32, 3)\n", + "n_classes = 10\n", + "train_size = len(X_train)\n", + "\n", + "print(f'Train: {train_size} Val: {len(X_val)} Test: {len(X_test)} Input shape: {input_shape}')\n", + "\n", + "fig, axes = plt.subplots(2, 5, figsize=(12, 4))\n", + "for i, ax in enumerate(axes.flat):\n", + " ax.imshow(X_train[i])\n", + " ax.set_title(str(np.argmax(Y_train[i])))\n", + " ax.axis('off')\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "markdown", + "id": "c5289d1e", + "metadata": {}, + "source": [ + "## Build data pipelines\n", + "\n", + "Images are normalised to [0, 1] and labels are one-hot encoded during loading. The data is shuffled with a fixed seed before the 75/25 train/validation split to ensure a balanced class distribution in both sets.\n", + "\n", + "The training and validation sets are wrapped in `tf.data` pipelines. Chaining `.shuffle` → `.batch` → `.prefetch` ensures the next batch is prepared on the CPU while the GPU trains on the current one. `tf.data.AUTOTUNE` lets TensorFlow select the number of parallel threads for `.prefetch` automatically at runtime.\n", + "\n", + "The test set remains as numpy arrays for direct use with sklearn and hls4ml." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a-4", + "metadata": {}, + "outputs": [], + "source": [ + "AUTOTUNE = tf.data.AUTOTUNE\n", + "batch_size = 128\n", + "\n", + "train_data = tf.data.Dataset.from_tensor_slices((X_train, Y_train)).shuffle(4096).batch(batch_size).prefetch(AUTOTUNE)\n", + "val_data = tf.data.Dataset.from_tensor_slices((X_val, Y_val)).batch(batch_size).prefetch(AUTOTUNE)\n", + "\n", + "print(f'Test set: X={X_test.shape}, Y={Y_test.shape}')" + ] + }, + { + "cell_type": "markdown", + "id": "4a-5", + "metadata": {}, + "source": [ + "## Define the quantized model\n", + "\n", + "We use QKeras to train a quantized CNN. Each convolutional block consists of a `QConv2D` layer with 6-bit quantized weights, followed by a standard `BatchNormalization` and a 6-bit quantized ReLU activation (`quantized_relu(6)`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a-6", + "metadata": {}, + "outputs": [], + "source": [ + "filters_per_conv_layer = [16, 16, 24]\n", + "neurons_per_dense_layer = [42, 64]\n", + "\n", + "x = x_in = Input(shape=input_shape)\n", + "\n", + "for i, f in enumerate(filters_per_conv_layer):\n", + " x = QConv2D(\n", + " int(f),\n", + " kernel_size=(3, 3),\n", + " kernel_quantizer='quantized_bits(6,0,alpha=1)',\n", + " bias_quantizer='quantized_bits(6,0,alpha=1)',\n", + " use_bias=True,\n", + " name=f'conv_{i}',\n", + " )(x)\n", + " x = BatchNormalization(name=f'bn_conv_{i}')(x)\n", + " x = QActivation('quantized_relu(6)', name=f'conv_act_{i}')(x)\n", + " x = MaxPooling2D(pool_size=(2, 2), name=f'pool_{i}')(x)\n", + "x = Flatten()(x)\n", + "\n", + "for i, n in enumerate(neurons_per_dense_layer):\n", + " x = QDense(\n", + " n,\n", + " kernel_quantizer='quantized_bits(6,0,alpha=1)',\n", + " name=f'dense_{i}',\n", + " use_bias=False,\n", + " )(x)\n", + " x = BatchNormalization(name=f'bn_dense_{i}')(x)\n", + " x = QActivation('quantized_relu(6)', name=f'dense_act_{i}')(x)\n", + "\n", + "x = Dense(n_classes, name='output_dense')(x)\n", + "x_out = Activation('softmax', name='output_softmax')(x)\n", + "\n", + "model = Model(inputs=[x_in], outputs=[x_out], name='qkeras_cnn')\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "id": "4a-8", + "metadata": {}, + "source": [ + "## Train the model\n", + "\n", + "**Note on accuracy**: the accuracy achieved in this notebook is lower than the one reported in the [hls4ml CNN paper](https://arxiv.org/abs/2101.05108), because we only train on the 73k-image training split rather than the 531k-image `extra` split used in the paper. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a-9", + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(\n", + " optimizer=keras.optimizers.Adam(learning_rate=5e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-7, amsgrad=True),\n", + " loss=keras.losses.CategoricalCrossentropy(),\n", + " metrics=['accuracy'],\n", + ")\n", + "callbacks = [\n", + " keras.callbacks.EarlyStopping(patience=15, restore_best_weights=True, verbose=1),\n", + " keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, verbose=1),\n", + "]\n", + "t0 = time.time()\n", + "model.fit(train_data, epochs=20, validation_data=val_data, callbacks=callbacks)\n", + "print(f'\\nTraining took {(time.time() - t0) / 60:.1f} minutes')\n", + "os.makedirs('../models', exist_ok=True)\n", + "model.save('../models/qkeras_cnn_part4a.keras')" + ] + }, + { + "cell_type": "markdown", + "id": "4a-10", + "metadata": {}, + "source": [ + "## Performance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a-11", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "\n", + "y_pred = model.predict(X_test)\n", + "print(f'QKeras accuracy: {accuracy_score(np.argmax(Y_test, axis=1), np.argmax(y_pred, axis=1)):.4f}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a-12", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn import metrics\n", + "\n", + "colors = ['#67001f', '#b2182b', '#d6604d', '#f4a582', '#fddbc7', '#d1e5f0', '#92c5de', '#4393c3', '#2166ac', '#053061']\n", + "\n", + "fig, ax = plt.subplots(figsize=(10, 10))\n", + "for i in range(n_classes):\n", + " fpr, tpr, _ = metrics.roc_curve(Y_test[:, i], y_pred[:, i])\n", + " auc = metrics.auc(fpr, tpr)\n", + " ax.plot(fpr, tpr, label=f'{i}, AUC = {auc*100:.1f}%', linewidth=1.5, c=colors[i])\n", + "\n", + "ax.set_xlabel('False Positive Rate')\n", + "ax.set_ylabel('True Positive Rate')\n", + "ax.set_xlim(0.01, 1.0)\n", + "ax.set_ylim(0.5, 1.1)\n", + "ax.semilogx()\n", + "ax.legend(loc='lower right')" + ] + }, + { + "cell_type": "markdown", + "id": "4a-13", + "metadata": {}, + "source": [ + "## Convolutions in hls4ml\n", + "\n", + "hls4ml supports two I/O modes for neural networks:\n", + "\n", + "- **`io_parallel`**: All inputs arrive simultaneously. Suitable for small models, when all activations fit into registers.\n", + "- **`io_stream`**: Data flows through the network one element at a time via FIFO buffers. Required for larger CNNs, when the full feature maps are too large to hold in registers. Shift registers maintain a sliding window of `kernel_height − 1` rows, feeding the convolution kernel one pixel at a time.\n", + "\n", + "See the [hls4ml documentation](https://fastmachinelearning.org/hls4ml/concepts.html) for more details.\n", + "\n", + "![Conv2D stream implementation](../images/part4a_conv2d_animation.gif)\n", + "\n", + "**Note on softmax precision:** using `auto` precision for the output of the last dense layer can produce accumulators wider than the softmax look-up tables can handle. We cap this manually with `fixed<16,6,RND,SAT>`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a-14", + "metadata": {}, + "outputs": [], + "source": [ + "import hls4ml\n", + "\n", + "hls_config = hls4ml.utils.config_from_keras_model(\n", + " model, granularity='name', backend='Vitis', default_precision='fixed<16,6>', max_precision='fixed<18,8>'\n", + ")\n", + "hls_config['LayerName']['output_dense']['Precision']['result'] = 'fixed<16,6,RND,SAT>'\n", + "plotting.print_dict(hls_config)\n", + "\n", + "hls_model = hls4ml.converters.convert_from_keras_model(\n", + " model,\n", + " hls_config=hls_config,\n", + " backend='Vitis',\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_cnn_part4a',\n", + " part='xcu200-fsgd2104-2-e',\n", + " io_type='io_stream',\n", + ")\n", + "hls_model.compile()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a-15", + "metadata": {}, + "outputs": [], + "source": [ + "hls4ml.utils.plot_model(hls_model, show_shapes=True, show_precision=True, to_file=None)" + ] + }, + { + "cell_type": "markdown", + "id": "4a-16", + "metadata": {}, + "source": [ + "## Bit-accurate emulation\n", + "\n", + "We run `hls_model.predict` on a subset of the test set (the full set takes a long time CNNs) to confirm that the fixed-point hls4ml model achieves satisfactory performance, before kicing off FPGA synthesis." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a-17", + "metadata": {}, + "outputs": [], + "source": [ + "X_test_reduced = np.ascontiguousarray(X_test[:3000])\n", + "Y_test_reduced = np.array(Y_test[:3000])\n", + "\n", + "y_pred_reduced = model.predict(X_test_reduced)\n", + "y_pred_hls = hls_model.predict(X_test_reduced)\n", + "\n", + "print(f'QKeras accuracy : {accuracy_score(np.argmax(Y_test_reduced, axis=1), np.argmax(y_pred_reduced, axis=1)):.4f}')\n", + "print(f'hls4ml accuracy : {accuracy_score(np.argmax(Y_test_reduced, axis=1), np.argmax(y_pred_hls, axis=1)):.4f}')" + ] + }, + { + "cell_type": "markdown", + "id": "4a-19", + "metadata": {}, + "source": [ + "## Synthesis\n", + "\n", + "Run Vitis HLS C-synthesis and Vivado RTL synthesis.\n", + "\n", + "**This can take around an hour.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a-20", + "metadata": {}, + "outputs": [], + "source": [ + "hls_model.build(csim=False, synth=True, vsynth=True)" + ] + }, + { + "cell_type": "markdown", + "id": "4a-21", + "metadata": {}, + "source": [ + "## Reports\n", + "\n", + "Extract latency and resource numbers from the HLS and Vivado synthesis reports." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a-22", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import pprint\n", + "\n", + "\n", + "def getReports(indir):\n", + " data = {}\n", + " report_vsynth = Path(f'{indir}/vivado_synth.rpt')\n", + " report_csynth = Path(f'{indir}/myproject_prj/solution1/syn/report/myproject_csynth.rpt')\n", + " if not (report_vsynth.is_file() and report_csynth.is_file()):\n", + " print(f'No reports found in {indir}')\n", + " return data\n", + " with report_vsynth.open() as f:\n", + " lines = np.array(f.readlines())\n", + " data['lut'] = int(lines[np.array(['CLB LUTs*' in l for l in lines])][0].split('|')[2])\n", + " data['ff'] = int(lines[np.array(['CLB Registers' in l for l in lines])][0].split('|')[2])\n", + " data['bram'] = float(lines[np.array(['Block RAM Tile' in l for l in lines])][0].split('|')[2])\n", + " data['dsp'] = int(lines[np.array(['DSPs' in l for l in lines])][0].split('|')[2])\n", + " data['lut_%'] = float(lines[np.array(['CLB LUTs*' in l for l in lines])][0].split('|')[6])\n", + " data['ff_%'] = float(lines[np.array(['CLB Registers' in l for l in lines])][0].split('|')[6])\n", + " data['bram_%'] = float(lines[np.array(['Block RAM Tile' in l for l in lines])][0].split('|')[6])\n", + " data['dsp_%'] = float(lines[np.array(['DSPs' in l for l in lines])][0].split('|')[6])\n", + " with report_csynth.open() as f:\n", + " lines = np.array(f.readlines())\n", + " lat_line = lines[np.argwhere(np.array(['Latency (cycles)' in l for l in lines])).flatten()[0] + 3]\n", + " data['latency_clks'] = int(lat_line.split('|')[2])\n", + " data['latency_us'] = float(lat_line.split('|')[2]) * 5.0 / 1000.0\n", + " data['latency_ii'] = int(lat_line.split('|')[6])\n", + " return data\n", + "\n", + "\n", + "pprint.pprint(getReports('../hls4ml_prjs/hls4ml_prj_cnn_part4a'))" + ] + }, + { + "cell_type": "markdown", + "id": "2680a6f2", + "metadata": {}, + "source": [ + "## Further reading\n", + "\n", + "For more details, see:\n", + "\n", + "- Aarrestad et al., \"Fast convolutional neural networks on FPGAs with hls4ml\", Mach. Learn. Sci. Tech. 2(4):045015 (2021), [arXiv:2101.05108](https://arxiv.org/abs/2101.05108)\n", + "- Ghielmetti et al., \"Real-time semantic segmentation on FPGAs for autonomous vehicles with hls4ml\", Mach. Learn. Sci. Tech. (2022), [arXiv:2205.07690](https://arxiv.org/abs/2205.07690)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "hls4ml-tutorial", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/part5_bdt.ipynb b/6_more_models/6a_bdt.ipynb similarity index 67% rename from part5_bdt.ipynb rename to 6_more_models/6a_bdt.ipynb index b486e65f..3369d968 100644 --- a/part5_bdt.ipynb +++ b/6_more_models/6a_bdt.ipynb @@ -6,17 +6,20 @@ "source": [ "\"conifer\"\n", "\n", - "In this notebook we will take the first steps with training a BDT with `xgboost`, then translating it to HLS code for FPGA with `conifer`\n", + "In this notebook we will take the first steps with training a boosted decision tree (BDT) with `xgboost`, then translating it to HLS code for FPGA inference with `conifer`.\n", "\n", - "Key concepts:\n", - "- model training\n", - "- model evaluation\n", - "- `conifer` configuration and conversion\n", - "- model emulation\n", - "- model synthesis\n", - "- accelerator creation\n", + "## What is a Boosted Decision Tree?\n", "\n", - "For some use cases, the Forest Processing Unit might be an easier entry point as no FPGA synthesis is required for supported boards. Read more about the FPU here: https://ssummers.web.cern.ch/conifer/fpu.html" + "A Boosted Decision Tree (BDT) is an ensemble learning method that builds a strong classifier by combining many shallow decision trees. Each tree is trained to correct the residual errors of the previous ones. `XGBoost` is a particularly efficient and widely used gradient boosting framework that adds regularisation and second-order gradient information to improve generalisation and training speed. BDTs are popular in high-energy physics because they train quickly, are interpretable, and are often competitive with deep neural networks on tabular data. Their tree-structured computation also maps naturally to FPGA hardware: each tree can be evaluated in parallel, making BDTs well-suited for low-latency trigger and online inference applications.\n", + "\n", + "## Key notebook parts\n", + "\n", + "- **Model training**: train a multi-class `XGBClassifier` on the jet tagging dataset and compare its accuracy to the Keras/PyTorch baseline from Part 1\n", + "- **Model evaluation**: measure classification performance using ROC and accuracy\n", + "- **`conifer` configuration and conversion**: configure the `xilinxhls` backend and convert the trained XGBoost model into `conifer`'s intermediate representation, which generates synthesisable HLS C++ code\n", + "- **Model emulation**: compile the generated HLS C++ on the CPU and run bit-accurate predictions to verify conversion correctness and numerical precision before FPGA synthesis\n", + "- **Model synthesis**: run Vitis HLS C Synthesis followed by Vivado RTL synthesis\n", + "- **Accelerator creation**: configure a board-specific deployment target and build a complete bitfile for a `pynq-z2` board, ready for on-device inference\n" ] }, { @@ -27,6 +30,9 @@ "source": [ "import xgboost as xgb\n", "import matplotlib.pyplot as plt\n", + "import sys\n", + "\n", + "sys.path.append('..')\n", "import plotting\n", "import numpy as np\n", "from scipy.special import softmax\n", @@ -34,21 +40,29 @@ "import conifer\n", "import json\n", "import os\n", - "import sys\n", - "\n", - "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']\n", "\n", - "# enable more output from conifer\n", + "# Enable more outputs from conifer\n", "import logging\n", "\n", "logging.basicConfig(stream=sys.stdout, level=logging.WARNING)\n", "logger = logging.getLogger('conifer')\n", "logger.setLevel('DEBUG')\n", "\n", - "# create a random seed at we use to make the results repeatable\n", + "# Create a random seed at we use to make the results repeatable\n", "seed = int('hls4ml-tutorial'.encode('utf-8').hex(), 16) % 2**31\n", "\n", - "print(f'Using conifer version {conifer.__version__}')" + "print(f'Using conifer version {conifer.__version__}')\n", + "\n", + "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "MODEL_TYPE = 'keras' # set to 'pytorch' if you used the PyTorch notebook in Part 1" ] }, { @@ -59,7 +73,7 @@ "\n", "Load the jet tagging dataset.\n", "\n", - "**Note**: you need to run part1 first." + "**Note**: you need to run part 1 first to generate the dataset files." ] }, { @@ -68,11 +82,11 @@ "metadata": {}, "outputs": [], "source": [ - "X_train_val = np.load('X_train_val.npy')\n", - "X_test = np.load('X_test.npy')\n", - "y_train_val_one_hot = np.load('y_train_val.npy')\n", - "y_test_one_hot = np.load('y_test.npy')\n", - "classes = np.load('classes.npy', allow_pickle=True)" + "X_train_val = np.load('../data/jet-tagging/X_train_val.npy')\n", + "X_test = np.load('../data/jet-tagging/X_test.npy')\n", + "y_train_val_one_hot = np.load('../data/jet-tagging/y_train_val.npy')\n", + "y_test_one_hot = np.load('../data/jet-tagging/y_test.npy')\n", + "classes = np.load('../data/jet-tagging/classes.npy', allow_pickle=True)" ] }, { @@ -131,32 +145,57 @@ "outputs": [], "source": [ "from sklearn.metrics import accuracy_score\n", - "from tensorflow.keras.models import load_model\n", "\n", - "# load the KERAS model from part 1\n", - "model_ref = load_model('model_1/KERAS_check_best_model.h5')\n", - "y_ref = model_ref.predict(X_test)\n", - "\n", - "# compute predictions of the xgboost model\n", + "if MODEL_TYPE == 'keras':\n", + " from tensorflow.keras.models import load_model\n", + "\n", + " model_ref = load_model('../models/keras_model_part1.h5')\n", + " y_ref = model_ref.predict(X_test)\n", + "\n", + "elif MODEL_TYPE == 'pytorch':\n", + " import torch\n", + " import torch.nn as nn\n", + "\n", + " class JetTagger(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(16, 64)\n", + " self.fc2 = nn.Linear(64, 32)\n", + " self.fc3 = nn.Linear(32, 32)\n", + " self.output = nn.Linear(32, 5)\n", + "\n", + " def forward(self, x):\n", + " x = torch.relu(self.fc1(x))\n", + " x = torch.relu(self.fc2(x))\n", + " x = torch.relu(self.fc3(x))\n", + " return torch.softmax(self.output(x), dim=1)\n", + "\n", + " model_ref = JetTagger()\n", + " model_ref.load_state_dict(torch.load('../models/pytorch_weights_part1.pt'))\n", + " model_ref.eval()\n", + " with torch.no_grad():\n", + " y_ref = model_ref(torch.FloatTensor(X_test)).numpy()\n", + "\n", + "# Compute predictions of the xgboost model\n", "y_xgb = clf.predict_proba(X_test)\n", - "print(f'Accuracy baseline: {accuracy_score(np.argmax(y_test_one_hot, axis=1), np.argmax(y_ref, axis=1)):.5f}')\n", + "print(f'Accuracy {MODEL_TYPE}: {accuracy_score(np.argmax(y_test_one_hot, axis=1), np.argmax(y_ref, axis=1)):.5f}')\n", "print(f'Accuracy xgboost: {accuracy_score(np.argmax(y_test_one_hot, axis=1), np.argmax(y_xgb, axis=1)):.5f}')\n", "\n", "fig, ax = plt.subplots(figsize=(9, 9))\n", "_ = plotting.makeRoc(y_test_one_hot, y_ref, classes, linestyle='--')\n", - "plt.gca().set_prop_cycle(None) # reset the colors\n", + "plt.gca().set_prop_cycle(None)\n", "_ = plotting.makeRoc(y_test_one_hot, y_xgb, classes, linestyle='-')\n", "\n", - "# add a legend\n", "from matplotlib.lines import Line2D\n", - "\n", - "lines = [\n", - " Line2D([0], [0], ls='--'),\n", - " Line2D([0], [0], ls='-'),\n", - "]\n", "from matplotlib.legend import Legend\n", "\n", - "leg = Legend(ax, lines, labels=['part1 Keras', 'xgboost'], loc='lower right', frameon=False)\n", + "leg = Legend(\n", + " ax,\n", + " [Line2D([0], [0], ls='--'), Line2D([0], [0], ls='-')],\n", + " labels=[f'part1 {MODEL_TYPE}', 'xgboost'],\n", + " loc='lower right',\n", + " frameon=False,\n", + ")\n", "ax.add_artist(leg)" ] }, @@ -170,7 +209,7 @@ "\n", "We will print the configuration, modify it, and print it again. The modifications are:\n", "- set the `OutputDirectory` to something descriptive\n", - "- set the `XilinxPart` to the part number of the FPGA on the Alveo U50" + "- set the `XilinxPart` to the part number of the FPGA on the Alveo U250" ] }, { @@ -181,16 +220,16 @@ "source": [ "cfg = conifer.backends.xilinxhls.auto_config()\n", "\n", - "# print the config\n", + "# Print the config\n", "print('Default Configuration\\n' + '-' * 50)\n", "plotting.print_dict(cfg)\n", "print('-' * 50)\n", "\n", - "# modify the config\n", - "cfg['OutputDir'] = 'model_5/'\n", - "cfg['XilinxPart'] = 'xcu250-figd2104-2L-e'\n", + "# Set output directory and target device\n", + "cfg['OutputDir'] = '../hls4ml_prjs/conifer_prj_bdt_part6a'\n", + "cfg['XilinxPart'] = 'xcu200-fsgd2104-2-e'\n", "\n", - "# print the config again\n", + "# Print the config again (to verify change)\n", "print('Modified Configuration\\n' + '-' * 50)\n", "plotting.print_dict(cfg)\n", "print('-' * 50)" @@ -220,14 +259,17 @@ "metadata": {}, "outputs": [], "source": [ - "# convert the model to the conifer representation\n", + "# Convert the model to the conifer representation\n", "conifer_model = conifer.converters.convert_from_xgboost(clf, cfg)\n", - "# print the help to see the API on the conifer_model\n", + "\n", + "# Print the help to see the API of the conifer_model\n", "help(conifer_model)\n", - "# write the project (writing HLS project to disk)\n", + "\n", + "# Write the project (writing HLS project to disk)\n", "conifer_model.write()\n", - "# save the conifer model - we can load this again later\n", - "clf.save_model('model_5/xgboost_model.json')" + "\n", + "# Save the xgboost model alongside the conifer project\n", + "clf.save_model('../hls4ml_prjs/conifer_prj_bdt_part6a/xgboost_model.json')" ] }, { @@ -237,10 +279,10 @@ "## Explore\n", "Browse the files in the newly created project directory to take a look at the HLS code.\n", "\n", - "The output of `!tree model_5` is:\n", + "The output of `!tree ../hls4ml_prjs/conifer_prj_bdt_part6a` is:\n", "\n", "```\n", - "model_5/\n", + "conifer_prj_bdt_part6a/\n", "├── bridge.cpp\n", "├── build_hls.tcl\n", "├── firmware\n", @@ -306,29 +348,27 @@ "source": [ "y_hls_proba = softmax(y_hls) # compute class probabilities from the raw predictions\n", "\n", - "print(f'Accuracy baseline: {accuracy_score(np.argmax(y_test_one_hot, axis=1), np.argmax(y_ref, axis=1)):.5f}')\n", + "print(f'Accuracy {MODEL_TYPE}: {accuracy_score(np.argmax(y_test_one_hot, axis=1), np.argmax(y_ref, axis=1)):.5f}')\n", "print(f'Accuracy xgboost: {accuracy_score(np.argmax(y_test_one_hot, axis=1), np.argmax(y_xgb, axis=1)):.5f}')\n", "print(f'Accuracy conifer: {accuracy_score(np.argmax(y_test_one_hot, axis=1), np.argmax(y_hls_proba, axis=1)):.5f}')\n", "\n", - "\n", "fig, ax = plt.subplots(figsize=(9, 9))\n", "_ = plotting.makeRoc(y_test_one_hot, y_ref, classes, linestyle='--')\n", - "plt.gca().set_prop_cycle(None) # reset the colors\n", + "plt.gca().set_prop_cycle(None)\n", "_ = plotting.makeRoc(y_test_one_hot, y_xgb, classes, linestyle=':')\n", - "plt.gca().set_prop_cycle(None) # reset the colors\n", + "plt.gca().set_prop_cycle(None)\n", "_ = plotting.makeRoc(y_test_one_hot, y_hls_proba, classes, linestyle='-')\n", "\n", - "# add a legend\n", "from matplotlib.lines import Line2D\n", - "\n", - "lines = [\n", - " Line2D([0], [0], ls='--'),\n", - " Line2D([0], [0], ls=':'),\n", - " Line2D([0], [0], ls='-'),\n", - "]\n", "from matplotlib.legend import Legend\n", "\n", - "leg = Legend(ax, lines, labels=['part1 Keras', 'xgboost', 'conifer'], loc='lower right', frameon=False)\n", + "leg = Legend(\n", + " ax,\n", + " [Line2D([0], [0], ls='--'), Line2D([0], [0], ls=':'), Line2D([0], [0], ls='-')],\n", + " labels=[f'part1 {MODEL_TYPE}', 'xgboost', 'conifer'],\n", + " loc='lower right',\n", + " frameon=False,\n", + ")\n", "ax.add_artist(leg)" ] }, @@ -337,11 +377,11 @@ "metadata": {}, "source": [ "## Build\n", - "Now we'll run the Vitis HLS and Vivado synthesis. HLS C Synthesis compiles our C++ to RTL, performing scheduling and resource mapping. Vivado synthesis synthesizes the RTL from the previous step into a netlist, and produces a more realistic resource estimation. The latency can't change during Vivado synthesis, it's fixed in the RTL description.\n", + "Now we'll run the Vitis HLS and Vivado synthesis. HLS C Synthesis compiles our C++ to RTL, performing scheduling and resource mapping. Vivado synthesis synthesizes the RTL from the previous step into a netlist, and produces a more realistic resource estimation. \n", "\n", "After the build completes we can also browse the new log files and reports that are generated.\n", "\n", - "**Warning**: this step might take around 10 minutes" + "**This step takes around 10 minutes.**" ] }, { @@ -397,7 +437,7 @@ "outputs": [], "source": [ "pynq_model_cfg = conifer.backends.xilinxhls.auto_config()\n", - "pynq_model_cfg['OutputDir'] = 'model_5_pynq' # choose a new project directory\n", + "pynq_model_cfg['OutputDir'] = '../hls4ml_prjs/conifer_prj_bdt_part6a_pynq'\n", "pynq_model_cfg['ProjectName'] = 'conifer_jettag'\n", "pynq_model_cfg['AcceleratorConfig'] = {\n", " 'Board': 'pynq-z2', # choose a pynq-z2 board\n", @@ -444,7 +484,7 @@ "source": [ "### Load the model\n", "\n", - "We load the JSON for the conifer model we previously used, applying the new configuration just defined. We'll see that the FPGA part specified by the board overrides the `XilinxPart` specified in the default." + "We load the JSON for the conifer model we previously saved, applying the new configuration just defined. We'll see that the FPGA part specified by the board overrides the `XilinxPart` specified in the default." ] }, { @@ -453,7 +493,7 @@ "metadata": {}, "outputs": [], "source": [ - "pynq_model = conifer.model.load_model('model_5/my_prj.json', new_config=pynq_model_cfg)\n", + "pynq_model = conifer.model.load_model('../hls4ml_prjs/conifer_prj_bdt_part6a/my_prj.json', new_config=pynq_model_cfg)\n", "pynq_model.write()" ] }, @@ -465,11 +505,11 @@ "\n", "Now we run `build` again, running HLS Synthesis, Logic Synthesis and Place & Route, finally producing a bitfile and an archive of files that we'll need to run inference on the pynq-z2 board. \n", "\n", - "**Warning**: this step might take around 20 minutes to complete.\n", + "**This step takes around 20 minutes.**\n", "\n", "The floorplan of the bitfile should like something like this, where the individual tree modules are highlighted in different colours:\n", "\n", - "" + "" ] }, { @@ -488,9 +528,9 @@ "## Inference on pynq-z2\n", "\n", "Running inference on the `pynq-z2` would look like this:\n", - "- download the `model_5/conifer_jettag.zip` archive from this notebook\n", - "- upload `conifer_jettag.zip` to the pynq-z2 device and unzip it\n", - "- start a jupyter notebook on the `pynq-z2` and run the following code:\n", + "- Download the `conifer_bdt_pynq/conifer_jettag.zip` archive from this notebook\n", + "- Upload `conifer_jettag.zip` to the pynq-z2 device and unzip it\n", + "- Start a jupyter notebook on the `pynq-z2` and run the following code:\n", "\n", "```\n", "import conifer\n", @@ -499,11 +539,21 @@ "y_pynq = accelerator.decision_function(X)\n", "```\n" ] + }, + { + "cell_type": "markdown", + "id": "6d464270", + "metadata": {}, + "source": [ + "## Further reading\n", + "\n", + "For more details, see: Summers, Di Guglielmo, Duarte et al., \"Fast inference of Boosted Decision Trees in FPGAs for particle physics\", JINST 15 P05026 (2020), [arXiv:2002.02534](https://arxiv.org/abs/2002.02534)" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "hls4ml-tutorial", "language": "python", "name": "python3" }, @@ -517,7 +567,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.10" + "version": "3.10.16" } }, "nbformat": 4, diff --git a/6_more_models/6b_symbolic_regression.ipynb b/6_more_models/6b_symbolic_regression.ipynb new file mode 100644 index 00000000..a12ad5b6 --- /dev/null +++ b/6_more_models/6b_symbolic_regression.ipynb @@ -0,0 +1,441 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6b-0", + "metadata": {}, + "source": [ + "# Part 6b: Symbolic Regression\n", + "\n", + "In this notebook we will train a **symbolic regression (SR)** model on the LHC jet tagging dataset and convert it to a low-latency, low-resource FPGA design with hls4ml.\n", + "\n", + "## What is Symbolic Regression?\n", + "\n", + "Symbolic regression is a machine learning technique that searches for a **mathematical expression** — a formula containing elementary operations (+, -, ×) and functions (sin, cos etc.) — that best fits the training data; for example:\n", + "\n", + "$$f(x) = \\sin(x_3 + 0.5 \\cdot x_{14}) \\cdot (x_2 - 1.2)$$\n", + "\n", + "### How is it trained?\n", + "\n", + "SR is typically solved with **genetic programming**: a population of candidate expressions evolves over many generations. At each step, expressions are mutated (e.g. a node is swapped for a different operator) and crossed over (subtrees are exchanged between two parent expressions). Candidate functions are selected for survival based on a fitness function that penalises both prediction error and expression complexity, driving the search towards simple, accurate formulas.\n", + "\n", + "### Why use SR for FPGA inference?\n", + "\n", + "Neural networks rely on repeated **multiply-accumulate (MAC)** operations arranged across many layers. This translates directly to a large number of DSP blocks and LUT resources on the FPGA, and the depth of the network drives up latency.\n", + "\n", + "Symbolic regression takes a different trade-off: it produces fewer but more complex operations — `sin`, `cos`, arithmetic — applied in a compact formula. Because there are far fewer operations overall, both latency and resource usage can drop substantially compared to a neural network of equivalent accuracy.\n", + "\n", + "These complex mathematical functions can also be efficiently approximated with **lookup tables**, which can further reduce the resource consumption, as shown in the rest of this notebook.\n", + "\n", + "One important caveat: SR search is combinatorial and scales poorly with the number of input features, so it is best suited to problems with a small-to-moderate input dimensionality, such as the 16-feature jet tagging task here.\n", + "\n", + "### PySR\n", + "\n", + "[PySR](https://github.com/MilesCranmer/PySR) is a high-performance Python library for symbolic regression based on genetic programming. Its search backend is written in Julia, which it manages automatically: on the first run inside a fresh environment, PySR will download and install Julia — **this can take 5–10 minutes**. Subsequent runs start immediately.\n", + "\n", + "## Key notebook parts\n", + "\n", + "- **Model training**: run PySR to find symbolic expressions for each of the five jet classes\n", + "- **hls4ml conversion**: convert the expressions into a synthesisable HLS C++ project, in two flavours:\n", + " - **Standard** (`hls_model`): uses the Vivado/Vitis HLS math library for `sin`/`cos`\n", + " - **LUT-approximated** (`hls_model_lut`): replaces `sin`/`cos` with lookup-table approximations to save FPGA resources\n", + "- **Performance comparison**: compare accuracy and ROC curves between the two HLS implementations\n", + "- **Synthesis**: run Vivado/Vitis HLS C-synthesis to estimate latency and resource usage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b-1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "import sympy\n", + "import matplotlib.pyplot as plt\n", + "import hls4ml\n", + "from scipy.special import softmax\n", + "from sklearn.metrics import roc_curve, auc, accuracy_score\n", + "from pysr import PySRRegressor\n", + "\n", + "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']\n", + "\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "id": "6b-2", + "metadata": {}, + "source": [ + "## Load the jet tagging dataset\n", + "\n", + "We load the preprocessed arrays saved by `1_getting_started/1a_train_keras.ipynb` or `1b_train_pytorch.ipynb`. Run either notebook first.\n", + "\n", + "PySR uses `L2MarginLoss`, a margin-based loss that expects class labels in $\\{-1, +1\\}$ rather than the one-hot $\\{0, 1\\}$ encoding used in earlier parts. We convert with $Y = 2Y_{\\text{one-hot}} - 1$." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b-3", + "metadata": {}, + "outputs": [], + "source": [ + "X_train_val = np.load('../data/jet-tagging/X_train_val.npy')\n", + "X_test = np.load('../data/jet-tagging/X_test.npy')\n", + "y_train_val = np.load('../data/jet-tagging/y_train_val.npy')\n", + "y_test = np.load('../data/jet-tagging/y_test.npy')\n", + "classes = np.load('../data/jet-tagging/classes.npy', allow_pickle=True)\n", + "\n", + "# Convert one-hot {0,1} → margin labels {-1,+1} required by L2MarginLoss\n", + "Y_train_val = 2 * y_train_val - 1\n", + "Y_test = 2 * y_test - 1\n", + "\n", + "print(f'Training set: {X_train_val.shape}, Test set: {X_test.shape}')" + ] + }, + { + "cell_type": "markdown", + "id": "6b-4", + "metadata": {}, + "source": [ + "## Limit the training set\n", + "\n", + "Genetic programming explores a combinatorially large space of expressions and evaluates each candidate on the full training set at every generation. Runtime therefore scales roughly linearly with the number of training samples. Using the full 580k-sample dataset would make each generation prohibitively slow. We limit to 8,000 samples — enough to guide the search towards good expressions while keeping the training fast." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b-5", + "metadata": {}, + "outputs": [], + "source": [ + "N_TRAIN = 8000\n", + "X_train = X_train_val[:N_TRAIN]\n", + "Y_train = Y_train_val[:N_TRAIN]\n", + "\n", + "print(f'X_train: {X_train.shape}, Y_train: {Y_train.shape}')\n", + "print(f'X_test: {X_test.shape}, Y_test: {Y_test.shape}')" + ] + }, + { + "cell_type": "markdown", + "id": "6b-6", + "metadata": {}, + "source": [ + "## Train SR with PySR\n", + "\n", + "We configure PySR with a restricted operator set to keep expressions compact and FPGA-friendly:\n", + "\n", + "- **`binary_operators`**: `+`, `-`, `*` — the basic arithmetic building blocks\n", + "- **`unary_operators`**: `sin` and a custom `sc(x) = sin(x)·cos(x)` shorthand\n", + "- **`constraints`** / **`nested_constraints`**: prevent deeply nested trigonometric calls. Without this restriction, expressions like `sin(sin(sc(x + 1.3)))` could appear — deeply nested functions inflate the number of hardware operations and increase latency without a proportional gain in accuracy.\n", + "- **`select_k_features=6`**: PySR internally selects the 6 most informative features out of the 16 available, reducing the search space\n", + "- **`loss='L2MarginLoss()'`**: $(1 - y \\cdot \\hat{y})^2$, a margin loss on the raw (pre-softmax) output\n", + "- **`timeout_in_seconds=600`**: the search stops after 10 minutes regardless of iteration count\n", + "\n", + "**Note:** if this is the first time running PySR in this environment, it will download and install Julia before starting the search. This is a one-time setup that takes approximately 5–10 minutes and is separate from the 10-minute training time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b-7", + "metadata": {}, + "outputs": [], + "source": [ + "model_pysr = PySRRegressor(\n", + " model_selection='accuracy',\n", + " niterations=20,\n", + " timeout_in_seconds=600,\n", + " maxsize=30,\n", + " select_k_features=6,\n", + " binary_operators=['+', '-', '*'],\n", + " unary_operators=['sin', 'sc(x)=sin(x)*cos(x)'],\n", + " complexity_of_operators={'+': 1, '-': 1, '*': 1, 'sin': 1, 'sc': 1},\n", + " constraints={'sin': 20, 'sc': 20},\n", + " nested_constraints={'sin': {'sin': 0, 'sc': 0}, 'sc': {'sin': 0, 'sc': 0}},\n", + " extra_sympy_mappings={'sc': lambda x: sympy.sin(x) * sympy.cos(x)},\n", + " loss='L2MarginLoss()',\n", + ")\n", + "model_pysr.fit(X_train, Y_train)" + ] + }, + { + "cell_type": "markdown", + "id": "74393c59", + "metadata": {}, + "source": [ + "**Note:** With the settings above the model typically achieves around **70% classification accuracy** on the test set. Training is kept deliberately short so the tutorial runs in a reasonable time — increasing `timeout_in_seconds` or `niterations` will allow PySR to explore more of the expression space and find better-performing formulas." + ] + }, + { + "cell_type": "markdown", + "id": "6b-8", + "metadata": {}, + "source": [ + "## Extract expressions\n", + "\n", + "We extract the best expression found for each of the five jet classes and print them. Two string variants are prepared:\n", + "- **`expr`**: uses `sin`/`cos` directly — hls4ml calls the Vivado/Vitis HLS math library\n", + "- **`expr_lut`**: replaces them with `sin_lut`/`cos_lut` — hls4ml generates lookup-table approximations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b-9", + "metadata": {}, + "outputs": [], + "source": [ + "for i in range(5):\n", + " print(f'Tagger {i} ({classes[i]}) = {model_pysr.sympy()[i]}')\n", + " print('-' * 60)\n", + "\n", + "expr = [str(model_pysr.sympy()[i]) for i in range(5)]\n", + "expr_lut = [e.replace('sin', 'sin_lut').replace('cos', 'cos_lut') for e in expr]" + ] + }, + { + "cell_type": "markdown", + "id": "6b-12", + "metadata": {}, + "source": [ + "## Set up lookup-table approximations\n", + "\n", + "On FPGAs, computing `sin(x)` or `cos(x)` exactly requires a multi-cycle IP core that can consume significant DSP and LUT resources. An alternative is to pre-compute a table of values and approximate the function by looking up the nearest entry — a **lookup table (LUT)** (but, not to be confused with look-up tables as hardware primitives on AMD/Xilinx FPGAs).\n", + "\n", + "hls4ml's `init_pysr_lut_functions` registers custom function names (`sin_lut`, `cos_lut`) and associates each with a table specification:\n", + "\n", + "- **`range_start` / `range_end`**: the interval over which the table is populated. Values outside this range saturate to the nearest endpoint. The range `[-8, 8]` covers roughly 1.3 full periods of sin/cos and matches the typical argument range in these expressions.\n", + "- **`N`**: the number of table entries. With `N=256`, the interval `[-8, 8]` is divided into 256 equal steps of width $16/256 = 0.0625$, giving a maximum approximation error of about $\\cos(0.03) - 1 \\approx 5 \\times 10^{-4}$.\n", + "\n", + "Increasing `N` improves accuracy but increases BRAM usage; narrowing the range also improves accuracy if you know the arguments will be small." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b-13", + "metadata": {}, + "outputs": [], + "source": [ + "from hls4ml.utils.symbolic_utils import init_pysr_lut_functions\n", + "\n", + "function_definitions = [\n", + " 'sin_lut(x) = math_lut(sin, x, N=256, range_start=-8, range_end=8)',\n", + " 'cos_lut(x) = math_lut(cos, x, N=256, range_start=-8, range_end=8)',\n", + "]\n", + "init_pysr_lut_functions(init_defaults=True, function_definitions=function_definitions)\n", + "\n", + "lut_functions = {\n", + " 'sin_lut': {'math_func': 'sin', 'range_start': -8, 'range_end': 8, 'table_size': 256},\n", + " 'cos_lut': {'math_func': 'cos', 'range_start': -8, 'range_end': 8, 'table_size': 256},\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "6b-14", + "metadata": {}, + "source": [ + "## Parse expressions to sympy\n", + "\n", + "hls4ml expects sympy expression objects. We parse both string lists back into sympy format." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b-15", + "metadata": {}, + "outputs": [], + "source": [ + "for i in range(len(expr)):\n", + " expr[i] = sympy.parsing.sympy_parser.parse_expr(expr[i])\n", + " expr_lut[i] = sympy.parsing.sympy_parser.parse_expr(expr_lut[i])" + ] + }, + { + "cell_type": "markdown", + "id": "6b-16", + "metadata": {}, + "source": [ + "## Convert to hls4ml\n", + "\n", + "`convert_from_symbolic_expression` takes the list of sympy expressions (one per output class) and generates a Vivado/Vitis HLS C++ project. `n_symbols=16` tells hls4ml that the input has 16 features (`x0`–`x15`), matching the jet tagging dataset.\n", + "\n", + "**Vivado/Vitis HLS paths:** if `hls_model.compile()` raises an error about missing header files or shared libraries, you need to point hls4ml at your local Vivado/Vitis HLS installation. Uncomment and adjust the `HLS_INCLUDE_PATH` / `HLS_LIBS_PATH` lines in the cell below and pass them as arguments. The paths will differ depending on your Vivado/Vitis HLS version and installation location." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b-17", + "metadata": {}, + "outputs": [], + "source": [ + "# Uncomment and adjust these paths if hls_model.compile() raises errors\n", + "# about missing header files or shared libraries.\n", + "# HLS_INCLUDE_PATH = '/tools/Xilinx/Vitis_HLS/2024.1/include'\n", + "# HLS_LIBS_PATH = '/tools/Xilinx/Vitis_HLS/2024.1/lnx64'\n", + "\n", + "hls_model = hls4ml.converters.convert_from_symbolic_expression(\n", + " expr,\n", + " n_symbols=16,\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_sr_part6b',\n", + " precision='ap_fixed<16,6>',\n", + " part='xcu200-fsgd2104-2-e',\n", + " # hls_include_path=HLS_INCLUDE_PATH,\n", + " # hls_libs_path=HLS_LIBS_PATH,\n", + ")\n", + "hls_model.compile()\n", + "\n", + "hls_model_lut = hls4ml.converters.convert_from_symbolic_expression(\n", + " expr_lut,\n", + " n_symbols=16,\n", + " output_dir='../hls4ml_prjs/hls4ml_prj_sr_lut_part6b',\n", + " precision='ap_fixed<16,6>',\n", + " part='xcu200-fsgd2104-2-e',\n", + " # hls_include_path=HLS_INCLUDE_PATH,\n", + " # hls_libs_path=HLS_LIBS_PATH,\n", + " lut_functions=lut_functions,\n", + ")\n", + "hls_model_lut.compile()" + ] + }, + { + "cell_type": "markdown", + "id": "6b-20", + "metadata": {}, + "source": [ + "## Compare performance on the test set\n", + "\n", + "We run both HLS models over the full test set and compare accuracy and ROC-AUC against the PySR expressions evaluated in floating point." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b-21", + "metadata": {}, + "outputs": [], + "source": [ + "Y_hls = softmax(hls_model.predict(np.ascontiguousarray(X_test)), axis=1)\n", + "Y_hls_lut = softmax(hls_model_lut.predict(np.ascontiguousarray(X_test)), axis=1)\n", + "\n", + "y_true = np.argmax(y_test, axis=1)\n", + "\n", + "print(f'HLS accuracy: {accuracy_score(y_true, np.argmax(Y_hls, axis=1)):.4f}')\n", + "print(f'HLS LUT accuracy: {accuracy_score(y_true, np.argmax(Y_hls_lut, axis=1)):.4f}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b-22", + "metadata": {}, + "outputs": [], + "source": [ + "color = ['blue', 'orange', 'green', 'red', 'purple']\n", + "fig, ax = plt.subplots(figsize=(10, 8))\n", + "\n", + "for Y_pred, label, ls in [(Y_hls, 'HLS', '-'), (Y_hls_lut, 'HLS LUT', '--')]:\n", + " for c, cls in enumerate(classes):\n", + " fpr, tpr, _ = roc_curve(y_test[:, c], Y_pred[:, c])\n", + " ax.plot(\n", + " tpr,\n", + " fpr,\n", + " ls=ls,\n", + " color=color[c],\n", + " label=f'{cls}, AUC={auc(fpr, tpr):.2f}' if label == 'HLS' else '_nolegend_',\n", + " lw=1.5,\n", + " )\n", + "\n", + "from matplotlib.lines import Line2D\n", + "from matplotlib.legend import Legend\n", + "\n", + "style_lines = [Line2D([0], [0], ls=ls, color='k') for ls in ['-', '--']]\n", + "leg1 = ax.legend(loc='lower right', fontsize=10, frameon=False)\n", + "leg2 = Legend(ax, style_lines, ['HLS', 'HLS LUT'], loc='center right', frameon=False)\n", + "ax.add_artist(leg1)\n", + "ax.add_artist(leg2)\n", + "\n", + "ax.set_yscale('log')\n", + "ax.set_xlabel('True positive rate', size=13)\n", + "ax.set_ylabel('False positive rate', size=13)\n", + "ax.set_xlim(0, 1)\n", + "ax.set_ylim(0.001, 1)\n", + "ax.grid(True)" + ] + }, + { + "cell_type": "markdown", + "id": "6b-23", + "metadata": {}, + "source": [ + "## Synthesize\n", + "\n", + "Run Vivado/Vitis HLS C-synthesis to get latency and resource estimates.\n", + "\n", + "**This can take several minutes.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b-24", + "metadata": {}, + "outputs": [], + "source": [ + "hls_model.build(csim=False)\n", + "hls4ml.report.read_vivado_report('../hls4ml_prjs/hls4ml_prj_sr_part6b')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b-25", + "metadata": {}, + "outputs": [], + "source": [ + "hls_model_lut.build(csim=False)\n", + "hls4ml.report.read_vivado_report('../hls4ml_prjs/hls4ml_prj_sr_lut_part6b')" + ] + }, + { + "cell_type": "markdown", + "id": "5dd71647", + "metadata": {}, + "source": [ + "## Further reading\n", + "\n", + "For more details, see: Tsoi, Loncar, Dasu et al., \"SymbolNet: Neural Symbolic Regression with Adaptive Dynamic Pruning for Compression\", Mach. Learn. Sci. Tech. 6(1):015021 (2025), [arXiv:2401.09949](https://arxiv.org/abs/2401.09949)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "hls4ml-tutorial", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..810416fe --- /dev/null +++ b/TODO.md @@ -0,0 +1,21 @@ +1. Understand why hls4ml throws a warning when parsing Add nodes in QONNX (i.e. using bias in Brevitas model) + +2. Understand why QONNX needs softmax = legacy (if it's a bug in hls4ml, we should fix it; if not, we should add a note to the notebook) + +3. Once QKeras v3 support is merged to hls4ml main from Marius' branch, update environment.yml + +4. Add HGQ2 notebook (to replace current HGQ tutorial) in part 2 + +5. Once PQuant support is merged to hls4ml main, update environment.yml and add notebook to part 2 + +6. Add more models to `Advanced Models`: From Imperial HGQ CNN, GNN and MLP + +7. Add accelerator backend notebooks once the accelerator backends are merged + +8. Add detailed README.md with some progression figure indicating what tutorials to follow & update ToC + +9. Think about how to source Vivado/Vitis HLS - adding XILINX_HLS to the PATH may not work for all systems (e.g., some clusters use module) + +10. Add a notebook (e.g., 3c) on the different implementations of GEMV/Dense layers: Latency, Resource, da4ml + +11. Add a part 0 on basics of neural networks, FPGAs, etc., diff --git a/_toc.yml b/_toc.yml index c425fe27..2d905f1b 100644 --- a/_toc.yml +++ b/_toc.yml @@ -1,13 +1,13 @@ format: jb-book root: README.md chapters: - - file: part1_getting_started.ipynb - - file: part2_advanced_config.ipynb - - file: part3_compression.ipynb - - file: part4_quantization.ipynb - - file: part5_bdt.ipynb - - file: part6_cnns.ipynb - - file: part7a_bitstream.ipynb - - file: part7b_deployment.ipynb - - file: part7c_validation.ipynb - - file: part8_symbolic_regression.ipynb + - file: 1_getting_started/1a_train_keras.ipynb + - file: 1_getting_started/1b_train_pytorch.ipynb + - file: 1_getting_started/1c_hls4ml_synth.ipynb + - file: 2_quantization/2a_qkeras.ipynb + - file: 2_quantization/2b_brevitas.ipynb + - file: 3_advanced_config/3a_reuse_factor.ipynb + - file: 3_advanced_config/3b_profiling.ipynb + - file: 4_advanced_models/4a_qkeras_cnn_svhn.ipynb + - file: 6_more_models/6a_bdt.ipynb + - file: 6_more_models/6b_symbolic_regression.ipynb diff --git a/callbacks.py b/archived/callbacks.py similarity index 100% rename from callbacks.py rename to archived/callbacks.py diff --git a/nn_utils.py b/archived/nn_utils.py similarity index 100% rename from nn_utils.py rename to archived/nn_utils.py diff --git a/part4.1_HG_quantization.ipynb b/archived/part4.1_HG_quantization.ipynb similarity index 100% rename from part4.1_HG_quantization.ipynb rename to archived/part4.1_HG_quantization.ipynb diff --git a/part7a_bitstream.ipynb b/archived/part7a_bitstream.ipynb similarity index 100% rename from part7a_bitstream.ipynb rename to archived/part7a_bitstream.ipynb diff --git a/images/part7_block_design.png b/archived/part7a_block_design.png similarity index 100% rename from images/part7_block_design.png rename to archived/part7a_block_design.png diff --git a/images/part7_floorplan.png b/archived/part7a_floorplan.png similarity index 100% rename from images/part7_floorplan.png rename to archived/part7a_floorplan.png diff --git a/part7b_deployment.ipynb b/archived/part7b_deployment.ipynb similarity index 100% rename from part7b_deployment.ipynb rename to archived/part7b_deployment.ipynb diff --git a/part7c_validation.ipynb b/archived/part7c_validation.ipynb similarity index 100% rename from part7c_validation.ipynb rename to archived/part7c_validation.ipynb diff --git a/pruned_cnn/myproject_prj/solution1/syn/report/myproject_csynth.rpt b/archived/pruned_cnn/myproject_prj/solution1/syn/report/myproject_csynth.rpt similarity index 100% rename from pruned_cnn/myproject_prj/solution1/syn/report/myproject_csynth.rpt rename to archived/pruned_cnn/myproject_prj/solution1/syn/report/myproject_csynth.rpt diff --git a/pruned_cnn/vivado_synth.rpt b/archived/pruned_cnn/vivado_synth.rpt similarity index 100% rename from pruned_cnn/vivado_synth.rpt rename to archived/pruned_cnn/vivado_synth.rpt diff --git a/quantized_pruned_cnn/myproject_prj/solution1/syn/report/myproject_csynth.rpt b/archived/quantized_pruned_cnn/myproject_prj/solution1/syn/report/myproject_csynth.rpt similarity index 100% rename from quantized_pruned_cnn/myproject_prj/solution1/syn/report/myproject_csynth.rpt rename to archived/quantized_pruned_cnn/myproject_prj/solution1/syn/report/myproject_csynth.rpt diff --git a/quantized_pruned_cnn/vivado_synth.rpt b/archived/quantized_pruned_cnn/vivado_synth.rpt similarity index 100% rename from quantized_pruned_cnn/vivado_synth.rpt rename to archived/quantized_pruned_cnn/vivado_synth.rpt diff --git a/environment.yml b/environment.yml index 03ea888a..c941dce7 100644 --- a/environment.yml +++ b/environment.yml @@ -3,23 +3,22 @@ channels: - conda-forge dependencies: - python=3.10.16 - - notebook==6.4.12 - - jupyter_contrib_nbextensions - jupyterhub - - jupyter-book - - jsonschema-with-format-nongpl + - jupyterlab + - jupyter-book==2.1.5 - pydot==1.4.2 - graphviz==7.1.0 - - scikit-learn==1.2.2 - - tensorflow==2.14.0 - - tensorflow-datasets==4.8.3 - - webcolors - - widgetsnbextension==3.6.0 - - pip==23.0.1 + - onnx + - onnxscript + - onnxoptimizer + - pytorch + - tensorflow>=2.19 + - scikit-learn - pip: - - hls4ml[profiling,optimization,sr,HGQ,qkeras]==1.2.0 - - conifer==1.5 - - pysr==0.16.3 + - hls4ml[keras-v3,qkeras-v3,onnx,sr,profiling] @ git+https://github.com/makoeppel/hls4ml.git@feature/add_qkeras_v3 + - git+https://github.com/fastmachinelearning/qkerasV3.git + - brevitas + - qonnx + - conifer==1.8 + - pysr==1.5.4 - xgboost==1.7.5 - - zstd - - tensorflow-model-optimization diff --git a/images/reuse.png b/images/part3a_reuse_factor.png similarity index 100% rename from images/reuse.png rename to images/part3a_reuse_factor.png diff --git a/images/conv2d_animation.gif b/images/part4a_conv2d_animation.gif similarity index 100% rename from images/conv2d_animation.gif rename to images/part4a_conv2d_animation.gif diff --git a/images/test.png b/images/part4a_test_images.png similarity index 100% rename from images/test.png rename to images/part4a_test_images.png diff --git a/images/part5_floorplan.png b/images/part6a_bdt_floorplan.png similarity index 100% rename from images/part5_floorplan.png rename to images/part6a_bdt_floorplan.png diff --git a/images/conifer_v1.png b/images/part6a_conifer_logo.png similarity index 100% rename from images/conifer_v1.png rename to images/part6a_conifer_logo.png diff --git a/part1_getting_started.ipynb b/part1_getting_started.ipynb deleted file mode 100644 index 930fe537..00000000 --- a/part1_getting_started.ipynb +++ /dev/null @@ -1,407 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Part 1: Getting started" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.keras.utils import to_categorical\n", - "from sklearn.datasets import fetch_openml\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.preprocessing import LabelEncoder, StandardScaler\n", - "import numpy as np\n", - "\n", - "%matplotlib inline\n", - "seed = 0\n", - "np.random.seed(seed)\n", - "import tensorflow as tf\n", - "\n", - "tf.random.set_seed(seed)\n", - "import os\n", - "\n", - "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Fetch the jet tagging dataset from Open ML" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "data = fetch_openml('hls4ml_lhc_jets_hlf')\n", - "X, y = data['data'], data['target']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Let's print some information about the dataset\n", - "Print the feature names and the dataset shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "print(data['feature_names'])\n", - "print(X.shape, y.shape)\n", - "print(X[:5])\n", - "print(y[:5])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As you saw above, the `y` target is an array of strings, e.g. \\['g', 'w',...\\] etc.\n", - "We need to make this a \"One Hot\" encoding for the training.\n", - "Then, split the dataset into training and validation sets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "le = LabelEncoder()\n", - "y = le.fit_transform(y)\n", - "y = to_categorical(y, 5)\n", - "X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n", - "print(y[:5])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scaler = StandardScaler()\n", - "X_train_val = scaler.fit_transform(X_train_val)\n", - "X_test = scaler.transform(X_test)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "np.save('X_train_val.npy', X_train_val)\n", - "np.save('X_test.npy', X_test)\n", - "np.save('y_train_val.npy', y_train_val)\n", - "np.save('y_test.npy', y_test)\n", - "np.save('classes.npy', le.classes_)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Now construct a model\n", - "We'll use 3 hidden layers with 64, then 32, then 32 neurons. Each layer will use `relu` activation.\n", - "Add an output layer with 5 neurons (one for each class), then finish with Softmax activation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.keras.models import Sequential\n", - "from tensorflow.keras.layers import Dense, Activation, BatchNormalization\n", - "from tensorflow.keras.optimizers import Adam\n", - "from tensorflow.keras.regularizers import l1\n", - "from callbacks import all_callbacks" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model = Sequential()\n", - "model.add(Dense(64, input_shape=(16,), name='fc1', kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001)))\n", - "model.add(Activation(activation='relu', name='relu1'))\n", - "model.add(Dense(32, name='fc2', kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001)))\n", - "model.add(Activation(activation='relu', name='relu2'))\n", - "model.add(Dense(32, name='fc3', kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001)))\n", - "model.add(Activation(activation='relu', name='relu3'))\n", - "model.add(Dense(5, name='output', kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001)))\n", - "model.add(Activation(activation='softmax', name='softmax'))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train the model\n", - "We'll use Adam optimizer with categorical crossentropy loss.\n", - "The callbacks will decay the learning rate and save the model into a directory 'model_1'\n", - "The model isn't very complex, so this should just take a few minutes even on the CPU.\n", - "If you've restarted the notebook kernel after training once, set `train = False` to load the trained model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train = True\n", - "if train:\n", - " adam = Adam(lr=0.0001)\n", - " model.compile(optimizer=adam, loss=['categorical_crossentropy'], metrics=['accuracy'])\n", - " callbacks = all_callbacks(\n", - " stop_patience=1000,\n", - " lr_factor=0.5,\n", - " lr_patience=10,\n", - " lr_epsilon=0.000001,\n", - " lr_cooldown=2,\n", - " lr_minimum=0.0000001,\n", - " outputDir='model_1',\n", - " )\n", - " model.fit(\n", - " X_train_val,\n", - " y_train_val,\n", - " batch_size=1024,\n", - " epochs=10,\n", - " validation_split=0.25,\n", - " shuffle=True,\n", - " callbacks=callbacks.callbacks,\n", - " )\n", - "else:\n", - " from tensorflow.keras.models import load_model\n", - "\n", - " model = load_model('model_1/KERAS_check_best_model.h5')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Check performance\n", - "Check the accuracy and make a ROC curve" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import plotting\n", - "import matplotlib.pyplot as plt\n", - "from sklearn.metrics import accuracy_score\n", - "\n", - "y_keras = model.predict(X_test)\n", - "print(\"Accuracy: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_keras, axis=1))))\n", - "plt.figure(figsize=(9, 9))\n", - "_ = plotting.makeRoc(y_test, y_keras, le.classes_)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Convert the model to FPGA firmware with hls4ml\n", - "Now we will go through the steps to convert the model we trained to a low-latency optimized FPGA firmware with hls4ml.\n", - "First, we will evaluate its classification performance to make sure we haven't lost accuracy using the fixed-point data types. \n", - "Then we will synthesize the model with Vitis HLS and check the metrics of latency and FPGA resource usage.\n", - "\n", - "### Make an hls4ml config & model\n", - "The hls4ml Neural Network inference library is controlled through a configuration dictionary.\n", - "In this example we'll use the most simple variation, later exercises will look at more advanced configuration." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import hls4ml\n", - "\n", - "config = hls4ml.utils.config_from_keras_model(model, granularity='model', backend='Vitis')\n", - "print(\"-----------------------------------\")\n", - "print(\"Configuration\")\n", - "plotting.print_dict(config)\n", - "print(\"-----------------------------------\")\n", - "hls_model = hls4ml.converters.convert_from_keras_model(\n", - " model, hls_config=config, backend='Vitis', output_dir='model_1/hls4ml_prj', part='xcu250-figd2104-2L-e'\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's visualise what we created. The model architecture is shown, annotated with the shape and data types" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls4ml.utils.plot_model(hls_model, show_shapes=True, show_precision=True, to_file=None)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Compile, predict\n", - "Now we need to check that this model performance is still good. We compile the hls_model, and then use `hls_model.predict` to execute the FPGA firmware with bit-accurate emulation on the CPU." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls_model.compile()\n", - "X_test = np.ascontiguousarray(X_test)\n", - "y_hls = hls_model.predict(X_test)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Compare\n", - "That was easy! Now let's see how the performance compares to Keras:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Keras Accuracy: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_keras, axis=1))))\n", - "print(\"hls4ml Accuracy: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_hls, axis=1))))\n", - "\n", - "fig, ax = plt.subplots(figsize=(9, 9))\n", - "_ = plotting.makeRoc(y_test, y_keras, le.classes_)\n", - "plt.gca().set_prop_cycle(None) # reset the colors\n", - "_ = plotting.makeRoc(y_test, y_hls, le.classes_, linestyle='--')\n", - "\n", - "from matplotlib.lines import Line2D\n", - "\n", - "lines = [Line2D([0], [0], ls='-'), Line2D([0], [0], ls='--')]\n", - "from matplotlib.legend import Legend\n", - "\n", - "leg = Legend(ax, lines, labels=['keras', 'hls4ml'], loc='lower right', frameon=False)\n", - "ax.add_artist(leg)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Synthesize\n", - "Now we'll actually use Vitis HLS to synthesize the model. We can run the build using a method of our `hls_model` object.\n", - "After running this step, we can integrate the generated IP into a workflow to compile for a specific FPGA board.\n", - "In this case, we'll just review the reports that Vitis HLS generates, checking the latency and resource usage.\n", - "\n", - "**This can take several minutes.**\n", - "\n", - "While the C-Synthesis is running, we can monitor the progress looking at the log file by opening a terminal from the notebook home, and executing:\n", - "\n", - "`tail -f model_1/hls4ml_prj/vitis_hls.log`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "hls_model.build(csim=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Check the reports\n", - "Print out the reports generated by Vitis HLS. Pay attention to the Latency and the 'Utilization Estimates' sections" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls4ml.report.read_vivado_report('model_1/hls4ml_prj/')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Exercise\n", - "Since `ReuseFactor = 1` we expect each multiplication used in the inference of our neural network to use 1 DSP. Is this what we see? (Note that the Softmax layer should use 5 DSPs, or 1 per class)\n", - "Calculate how many multiplications are performed for the inference of this network...\n", - "(We'll discuss the outcome)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/part2_advanced_config.ipynb b/part2_advanced_config.ipynb deleted file mode 100644 index 974b51b9..00000000 --- a/part2_advanced_config.ipynb +++ /dev/null @@ -1,406 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Part 2: Advanced Configuration" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.keras.utils import to_categorical\n", - "from sklearn.datasets import fetch_openml\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.preprocessing import LabelEncoder, StandardScaler\n", - "from sklearn.metrics import accuracy_score\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "%matplotlib inline\n", - "import plotting\n", - "import os\n", - "\n", - "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Load the dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "X_train_val = np.load('X_train_val.npy')\n", - "X_test = np.ascontiguousarray(np.load('X_test.npy'))\n", - "y_train_val = np.load('y_train_val.npy')\n", - "y_test = np.load('y_test.npy', allow_pickle=True)\n", - "classes = np.load('classes.npy', allow_pickle=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Load the model\n", - "Load the model trained in 'part1_getting_started'. **Make sure you've run through that walkthrough first!**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.keras.models import load_model\n", - "\n", - "model = load_model('model_1/KERAS_check_best_model.h5')\n", - "y_keras = model.predict(X_test)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Make an hls4ml config & model\n", - "\n", - "When the parameter `granularity` is set to `'name'` in the `config_from_keras_model` function, hls4ml automatically chooses the fixed-point precision for the ouput variables and accumulators for each layer. The accumulators are internal variables used for accumulating values during matrix multiplications. \n", - "\n", - "This precision choice is **conservative**. It avoids overflow and truncation based solely on input bitwidths, without considering the actual input values. Once again, this approach can be overly conservative, especially when post-training quantization is employed or if the initial input bitwidth settings are relatively loose. In such cases, it is advisable to manually edit the configuration to explicitly set specific widths, potentially iteratively after profiling the data.\n", - "\n", - "In this notebook, we'll create a configuration with the finer granularity (`'name'`). When we print the config dictionary, you'll notice that an entry is created for each named Layer of the model and the types are set to `auto`. For example, for the first layer we have:\n", - "```\n", - "LayerName:\n", - " ...\n", - " fc1:\n", - " Trace: False\n", - " Precision:\n", - " weight: auto\n", - " bias: auto\n", - " result: auto\n", - " accum: auto\n", - " ReuseFactor: 1\n", - " ...\n", - "```\n", - "\n", - "In Part 1, all the parameters were set to the same default model precision. In this notebook instead, because of the `granularity='name'` and thus `'auto'` precision selection:\n", - "- `weight` and `bias` are set to the default model precision;\n", - "- `result` and `accum` are set to conservative bit-widths that avoid overflow and truncation.\n", - "\n", - "Later on, you will see that you can use this configuration as a template to start modifying things." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import hls4ml\n", - "\n", - "config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend='Vitis')\n", - "print(\"-----------------------------------\")\n", - "plotting.print_dict(config)\n", - "print(\"-----------------------------------\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Profiling\n", - "As you can see, we can choose the precision of _everything_ in our Neural Network. This is a powerful way to tune the performance, but it's also complicated. The tools in `hls4ml.model.profiling` can help you choose the right precision for your model. (That said, training your model with quantization built in can get around this problem, and that is introduced in Part 4. So, don't go too far down the rabbit hole of tuning your data types without first trying out quantization aware training with QKeras.)\n", - "\n", - "The first thing to try is to numerically profile your model. This method plots the distribution of the weights (and biases) as a box and whisker plot. The grey boxes show the values which can be represented with the data types used in the `hls_model`. Generally, you need the box to overlap completely with the whisker 'to the right' (large values) otherwise you'll get saturation & wrap-around issues. It can be okay for the box not to overlap completely 'to the left' (small values), but finding how small you can go is a matter of trial-and-error.\n", - "\n", - "Providing data, in this case just using the first 1000 examples for speed, will show the same distributions captured at the output of each layer." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "from hls4ml.model.profiling import numerical, get_ymodel_keras\n", - "\n", - "for layer in config['LayerName'].keys():\n", - " config['LayerName'][layer]['Trace'] = True\n", - "hls_model = hls4ml.converters.convert_from_keras_model(\n", - " model, hls_config=config, output_dir='model_1/hls4ml_prj_2', part='xcu250-figd2104-2L-e'\n", - ")\n", - "numerical(model=model, hls_model=hls_model, X=X_test[:1000])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Customize\n", - "Let's just try setting the precision of the first layer weights to something more narrow than 16 bits. Using fewer bits can save resources in the FPGA. After inspecting the profiling plot above, let's try 8 bits with 2 integer bit.\n", - "\n", - "**NOTE** Using `auto` precision can lead to undesired side effects. In case of this model, the bit width used for the output of the last fully connected layer is larger than can be reasonably represented with the look-up table in the softmax implementation. We therefore need to restrict it by hand to achieve proper results. \n", - "\n", - "Then create a new `HLSModel`, and display the profiling with the new config. This time, just display the weight profile by not providing any data '`X`'. Then create the `HLSModel` and display the architecture. Notice the box around the weights of the first layer reflects the different precision." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "config['LayerName']['fc1']['Precision']['weight'] = 'ap_fixed<8,2>'\n", - "config['LayerName']['output']['Precision']['result'] = 'fixed<16,6,RND,SAT>'\n", - "hls_model = hls4ml.converters.convert_from_keras_model(\n", - " model, hls_config=config, output_dir='model_1/hls4ml_prj_2', part='xcu250-figd2104-2L-e'\n", - ")\n", - "numerical(model=model, hls_model=hls_model)\n", - "hls4ml.utils.plot_model(hls_model, show_shapes=True, show_precision=True, to_file=None)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Trace\n", - "When we start using customised precision throughout the model, it can be useful to collect the output from each layer to find out when things have gone wrong. We enable this trace collection by setting `Trace = True` for each layer whose output we want to collect." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for layer in config['LayerName'].keys():\n", - " config['LayerName'][layer]['Trace'] = True\n", - "hls_model = hls4ml.converters.convert_from_keras_model(\n", - " model, hls_config=config, backend='Vitis', output_dir='model_1/hls4ml_prj_2', part='xcu250-figd2104-2L-e'\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Compile, trace, predict\n", - "Now we need to check that this model performance is still good after reducing the precision. We compile the `hls_model`, and now use the `hls_model.trace` method to collect the model output, and also the output for all the layers we enabled tracing for. This returns a dictionary with keys corresponding to the layer names of the model. Stored at that key is the array of values output by that layer, sampled from the provided data.\n", - "A helper function `get_ymodel_keras` will return the same dictionary for the Keras model.\n", - "\n", - "We'll just run the `trace` for the first 1000 examples, since it takes a bit longer and uses more memory than just running `predict`. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls_model.compile()\n", - "hls4ml_pred, hls4ml_trace = hls_model.trace(X_test[:1000])\n", - "keras_trace = get_ymodel_keras(model, X_test[:1000])\n", - "y_hls = hls_model.predict(X_test)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Inspect\n", - "Now we can print out, make plots, or do any other more detailed analysis on the output of each layer to make sure we haven't made the performance worse. And if we have, we can quickly find out where. Let's just print the output of the first layer, for the first sample, for both the Keras and hls4ml models." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Keras layer 'fc1', first sample:\")\n", - "print(keras_trace['fc1'][0])\n", - "print(\"hls4ml layer 'fc1', first sample:\")\n", - "print(hls4ml_trace['fc1'][0])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Compare\n", - "Let's see if we lost performance by using 8 bits for the weights of the first layer by inspecting the accuracy and ROC curve." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Keras Accuracy: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_keras, axis=1))))\n", - "print(\"hls4ml Accuracy: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_hls, axis=1))))\n", - "\n", - "fig, ax = plt.subplots(figsize=(9, 9))\n", - "_ = plotting.makeRoc(y_test, y_keras, classes)\n", - "plt.gca().set_prop_cycle(None) # reset the colors\n", - "_ = plotting.makeRoc(y_test, y_hls, classes, linestyle='--')\n", - "\n", - "from matplotlib.lines import Line2D\n", - "\n", - "lines = [Line2D([0], [0], ls='-'), Line2D([0], [0], ls='--')]\n", - "from matplotlib.legend import Legend\n", - "\n", - "leg = Legend(ax, lines, labels=['keras', 'hls4ml'], loc='lower right', frameon=False)\n", - "ax.add_artist(leg)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Profiling & Trace Summary\n", - "We lost a small amount of accuracy compared to when we used `ap_fixed<16,6>`, but in many cases this difference will be small enough to be worth the resource saving. You can choose how aggressive to go with quantization, but it's always sensible to make the profiling plots even with the default configuration. Layer-level `trace` is very useful for finding when you reduced the bitwidth too far, or when the default configuration is no good for your model.\n", - "\n", - "With this 'post training quantization', around 8-bits width generally seems to be the limit to how low you can go before suffering significant performance loss. In Part 4, we'll look at using 'training aware quantization' with QKeras to go much lower without losing much performance.\n", - "\n", - "## ReuseFactor\n", - "Now let's look at the other configuration parameter: `ReuseFactor`.\n", - "Recall that `ReuseFactor` is our mechanism for tuning the parallelism:" - ] - }, - { - "attachments": { - "reuse.png": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABu0AAAOmCAYAAADvj9j+AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH5AYCDTAgiLhLdwAAAB1pVFh0Q29tbWVudAAAAAAAQ3JlYXRlZCB3aXRoIEdJTVBkLmUHAAAgAElEQVR42uzdd5wV5b0/8O92YOlt6SjSRFFURAU1mhi7QWNsKdZoLIma3Bijvxi8ejVRo4klGsu1xtjbtXexRUUJinRpupRl6bC7sPX3h7Jw2F1YcNmz5f1+vXy9nOc8M2fOd56ZPZ6P80xKRUVFRQAAAAAAAABJk6oEAAAAAAAAkFxCOwAAAAAAAEgyoR0AAAAAAAAkmdAOAAAAAAAAkkxoBwAAAAAAAEkmtAMAAAAAAIAkE9oBAAAAAABAkgntAAAAAAAAIMmEdgAAAAAAAJBkQjsAAAAAAABIMqEdAAAAAAAAJJnQDgAAAAAAAJJMaAcAAAAAAABJJrQDAAAAAACAJBPaAQAAAAAAQJKlKwEAAACQbBdddFHcfffdCkGd+93vfhcXX3yxQgAADZ7QDgAAAEi6goKCWLp0qUJQ54qKihQBAGgUTI8JAAAAAAAASSa0AwAAAAAAgCQT2gEAAAAAAECSCe0AAAAAAAAgyYR2AAAAAAAAkGRCOwAAAAAAAEgyoR0AAAAAAAAkmdAOAAAAAAAAkixdCQAAAIDGokuXLnHeeecpBHHttddGYWGhQgAATYbQDgAAAGg0OnfuHGPGjFEI4pZbbhHaAQBNiukxAQAAAAAAIMmEdgAAAAAAAJBkQjsAAAAAAABIMqEdAAAAAAAAJFm6EgAAAAAA0BjkzpsfDz/2ZOVyVmZmnHryj6NN69aKAzR6QjsAAAAAABq8kpKS+NvN/4g5c79MaN9n7xExfPdhCgQ0eqbHBAAAAACgwfvnQ49VCewAmhKhHQAAAAAADdr4CZ/F8y++ohBAkya0AwAAAACgwVq+fEXcctudCgE0eUI7AAAAAAAapIqKirjlH3fFypWrFANo8oR2AAAAAAA0SM+98EpM+HSiQgDNgtAOAAAAAIAGZ9bsufHgQ48qBNBspCsBAAAAADR+ZeXlMW/egpi/YEFkpKdH//79ol3btrVef/GSJbFs2YpYtWpVZGVlRVZWZnTp3DnatWvbaGuyfPmKWJiXFwsWLoo1a9ZEdnaraNumTeTkdI3u3XLq9L2aYv2Sae3atfG3W/4RpWVligE0G0I7AAAAAGjgrrrmhli7dm3l8p577B5HHXFIREQUFRXF/Q8+EmPffi+KS0oq+/zq3DPjO/uN2uR2v/xqXrz+5tj4aNz4yF+8uNo+nTt3iu/sNzIO2G9UdO/ebbP7mp+/OG6+7c6Eth7du8fZZ55a68/75th34s2x71Yup6SkxCk/PSn6bd93s+vm5S2Kfz3yRHzyn09jzZo1Nfbr2qVzDN9jtzjmB0dEhw7tt+q4bIv68bV77v9XzJ+/IKHt4IMOjNx582PylGkKBDRJQjsAAAAAaOCmTpseRUXrA6iCgsI46ohDYtr0L+KGG2+NJUuXbtH2CgoK4qFHn4xXXn0jyisqNtl38eIl8cRTz8YTTz0bhx78vfjJScdFyxYtauy/tri4SqiSO2/+FoV2eYsWV9nGhE8nbjK0KygoiEefeCZefuX1Wt2dtSh/cbzw0qvx+ptj46Tjj40jDz+kQdSPiA8+HBevvTE2oa1Hj+5xyk9PjKuuuUGBgCZLaAcAAAAAjdDsOXPjqj9fH4VFRVu03tKly+KKq6+L3Hnzt/g9X3rl9fh4/IT4/W8viO369mkwtSgtLY0/Xfe3mDptxhavu3Ztcdz7wENRVFQUxx17dLOsX0OyeMmSuO3OexLa0tLS4sJf/iKysrIUCGjSUpUAAAAAABqXVatXx/9sRWC3ZOmy+MPlV21V4LTO4sVL4n/+fH0sys9vMPW4+74HNxnYpaakbHYbjzz+dEz4dGKzrF9DUV5eHjfecnsUFBQmtJ943DHRb/vtFAho8txpBwAAAACNzNKly2p8LTMjo9r28vLyuOnvt8ei/KrPXktJSYn9990nBg8aGL1794zCwqKYPmNmzJjxRUyeOj1KNnhWXkTE8uUr4k/X/i3+cs2VkZaa3PsCZs2eG6+89maV9j69e8Xoow6PXYfuFO3atY3ikpLIy8uP+QsWxFPPPBczZ82pss4/H3osdt1l50ipJuRrqvVrSJ58+rmYMnV6QtuQHQfF6KMOVxygWRDaAQAAAEAjN3jQgDjysIOjb5/ekZPTNVKrCYKef/GVmDR5apX2Th07xvnnnRU7DRmc0L77sF0iImLK1Olx1TU3xJo1axJe/yp3Xrz9zntx4Hf2S+pn/+Q/E6q07dBv+7jij79PmE4xKzMz+vTuGX1694w9h+8ez7/wctz/4CMJ682Z+2VMmjI1dh6yY7OpX0MxbfoX8egTTye0tWrVKn517lnVjmeApkhoBwAAAACNVEpKShwz+sg44bhjNnnH1tri4njymeeqtHfp3Dmu+9N/R+vW2TWuu+PggfGHS/4rrvrz9VFUlBg8PfHUs8kP7cZXDe2OPeaoTT7/LC01NX5w5GExZdqMGPfx+ITXvvwyt0po1xjqV1JSEjf9/Y56qfmvzz+nToO0wsKiuPGWf0R5eXlC+5mnnxxdOndyogPNhtAOAAAAABqpo444NH58wrGb7ff2O+/HqlWrq7T/4uenbDJwWmfwwAFx7llnxPU3/j2hfWHeopg3f0H07NE9aTVYmLeoamNFRa3WPfA7+1YJ7ebNX9Ao61dWVh7//nBcvdT8wjinTrd3x//eV2Xa0f1G7RP7jdrbSQ40K0I7AAAAAGiE2rdrF8f98Ae16jv2nfeqtI3ce88YtuvQWr/fiBF7RMcO7WPpsuUJ7Z9NnJTU0C4tLa1K2yuvvxW77bZrZKRv+ufPnYfsGOefd1ZCW7ecrs2qfsn21tvvxbvvf5DQ1rlzp/j56T9zkgPNjsmAAQCARmfexwVRUliuEBFR/PlnUZa3UCGq8casslhbWqEQ1Ri38JVYW1qkENDIHXHY96Nly5ab7VdSUhJfzJxdpX2vEcO36P3SUlOrXWf2nLlJrcOggf2rtE34dGL86dq/xoKFeZtct1WrlrH/viMT/hk4oH+zql8yLViYF3fd80BCW0pKSpx/3lmR3aqVkxxodoR2AABAozP7rdXx+mXzY9rzK5p9eFcy6fNYfvFvYvUdtwnvNvLPz8rinOdK4rlpwruNvTr3n3HlhyfF618+LLyDRmz77frWqt8XM2dHaWlplfZ+tVx/Q3379K7StvG0hvVtpx0HV9v+2cRJccFvfh+X/8818ezzL8fsOXOrPDOtKdUvPT0tunbt0mjGb2lpadx4yz9izZrE5/wd84MjYsjgQU5woFkyPSYAANAola6piBkvrIzZb6yK7b/bJvod2CYyWjXT/y+xvDzWvjs21r7/TmSN3C9ajj4m0nK6GSQRsXxNxN3/KYsnp5TFD3dMi+/vkBpZ6SkKExEFJSvjuVl3xJtfPRwH9j4x9u0xOrLSWyoMNCK9e/WsVb9F+fnVtt96+90RW3hJzJ03v0rbxtM91reRe4+IZ194ORYvXlL1T2RFRXw+aUp8PmlKRES0atUqdhw8MHbZeUjsMnSnWtWwsdQvPT09bvnrNVsVTG6ptNRv/53r4ceeqnIH4w79tovjf3S0kxtotoR2AABAoya824DwrkbCu5oJ76DxatOmda36rV5dUG37lGnT62Q/iouLk1qHDh3axxV//H388Yo/VxvcbaiwsDA+GT8hPhk/ISK+fnbaqH32iu/sNyr69O7Z6OuXmpoaqakN/3vQxM8nxzPPvpDQlpmZGRf88uxIT/eTNdB8uQICAABNgvBuA8K7Ggnvaia8ozoPPPBA5ObmJrRddNFFflRvZFbVEDrV2d/gktK632bplm2za5cuceUfL4l7//lwfDTuk6ioqN20yIsXL4lnnn0hnnn2hdhrxPA496zTIjs7u9HXr2F/TSmPm2+9o8oxOvVnJ0WP7r6vAM2bb1gAAECTIrzbgPCuRsK7mlUJ73qOjqw04V1z9ctf/jJWrlyZ0PbrX/9aaNfY/hyUlW3T7aek1v31s6Bgy4OyLl06x0W//mUsWJgXL7/6eoz7+D+Rtyi/1ut/+NHHsWrVqrjskt9GRkZGo65fw/56Ul7tlKB33/dg3H3fg5v/rldNoHvdDTdHSkpiHX960nFx5OGHuAAAjYpvWAAAQJMkvNuA8K5GwruaCe/Y8ktNeaxevTqhLTU1NVq3bq04SZbdOrva9ot+/as62X6PHnX/N2VrQrt1unfLiVN/9uM49Wc/jvkLFsZnEyfF55OnxOTJ02LlqlWbXHfylGlx2x13x/nn/aJR169Rfncr3fo7DsuqCVbr49l+AHVNaAcAADRpwrsNCO9qJLyrmfCO2po4cWIMGzYsoW3w4MExZcoUxUmy1tlVQ6eUlJTYbdjQyMzMbJjX5RUr62Q7Pbp3ix7du8WhB38vKioqYu6XuTHh089i/ITPYvKUadWu8+77H8bZZ55WWZvGUr/y8vL48qt59fJe2/Xt7cQC2AaEdgAAQLMgvNuA8K5GwruaCe+g8erQoX2VtoqKiliYtyj69O5VT396an/XU0lpaXwxc3ad70NKSkps17d3bNe3dxz9gyNiwcK8+Mcd98SkKVOr7OucuV/GwAH9G0z9aqO4uCR++/vL6uW9Hnnw7khLTXVyAdQxoR0AANCsCO82ILyrkfCuZsI7aHwG9O9Xbfu06V9scej0/IuvxNvv/juh7aQTjo1hu+xcuVzd1XL16oJYuXJVtG3bZrPvMWPGzCguLq7V/jz7/Evx7vsfJrT96twzo1fPHptdt3u3nLj097+J310yJubNX5Dw2sxZcypDu/quX1OXmpoap/z0xK1e/4F/PVolBP7uAftH716Jx3zIjoOd/ECjI7QDAACaJeHdBoR3NRLe1Ux4B41Hm9ato3evnvFVbuLUif/33IvxvQP3j9Ra3jG1YsXKeOjRJ2PNmjUJ7Rsvt25T/XMMZ82ZW6tw6q233631Z1uxYmXMnJV4V96ML2bWKrSLiMjKzIydhgyuEtoVFBQmrX5NXWpqahx1xKFbvf6UadPjo3HjE9pG7Ll7DN99mJMdaPzXSCUAAACas3Xh3euXzY9pz6+IksLy5luMb8K75Rf/JlbfcVuU5S00QL6xLrw757mSeG5aWawtrVCUb6wL76784KR4/cuHY21ZkaJAA7T3XsOrtC1YmBf//nBcrbfx2JPPVAmYWrVsGXvstmtCW5vWrasNsmbNmrPZ95g9Z268Obb2od0O/bav0jbuk/9sUW0WL1lapa1Pn15Jqx8AzZc77QAAAMKddwnceVejje+8O7h/amSmufMuwp130NAddvBB8cz/vRDFJSUJ7ffc/6/o2qVLjVNArvPm2Hfi1dffqtK+917DIyMjI6EtNTU1OrRvH0uWJoZhTz/7Quw2bGhsv13fat9j/oKFcc1fboyKitr/jxH9d6ga2n00bnx8+tnnsWst7uqbPWduTJo8pUr74IEDkla/rZWWnhaHHvy9bT6WevXs4Xl2ANuI0A4AAGADwrsNCO9qtC68e2pKWRwjvEvQlMO7ioqKyM3NjRkzZsSyZcuiV69e0a9fv+jSpctm1121alXMnj07cnNzo6ioKLp27Rrdu3ePfv361XpqPfg22rZtE4cf+v14+tkXEq9ny1fEZf99dZx1xilx4Hf2jZSUxGvZqtWr44677qv2jrJWrVrFscf8oNr3G77HsHj51TcS2goLC+PKq/8S55x1egwePCDatG79zfmxOl59/a34v+dfjNWrC7boc3Xp0jl69uheZXrL//nz9XH0Dw6PE350TKSnp1d7Po//z6dx0613xNq1ic/P69mje5Vn79V3/bZGRnp6/Py0nxnsAI2Y0A4AAKAawrsNCO9qtEx4V6PGFN6tXr06jjzyyIS2gw8+OC699NKIiFiyZEn87W9/i1tuuSWWL19eZf1evXrFH/7whzj99NOr3DHzwgsvxO233x4vvvhilGx0h866dU866aS4+OKLo1OnTpvcz7lz58Ypp5yS0DZo0KC4/fbba/1Z77333rj33nsrl1NTU+P666+P3Xbbbatqd9FFF8W4ceMq67ixL7/8Mg444ICEttdffz3S0tKcJElw4vE/jElTpsWML2Ym/s0rLY1bb//fuOf+B6Nvn97RokWLyEhPj4V5eTFv/sIoL69+6uhzf3F65HStPrTeb999qoR2ERErV62Ka66/MSIiOnfuFGVlZbFs2fJv9bl+8fNT449X/CmhraKiIp565vl4970Po1+/7aJn926Rk9MliotLIj9/cXww7pNYtCi/2u2ddvKPk14/AJonoR0AAMAmCO82ILyrkfCuZo0hvCstLY2xY8cmtOXm5sall14ajz/+eJx22mnVBlIb9j377LPjpptuijfeeCNycnJixYoVcd5558WDDz64yffOzc2N6667Lh566KF4/PHHY6+99qqxb2FhYZX9nDJlyhaFdrNnz66yjZdeemmrQ7tPP/20yvY2t89bMvUhdSs9PT1+e+F5cfn/XBMLFuZVeb2oaE1MnTajVts6/NDvx94jhtf4+uCBA2Lk3nvG+x/U/My3xYuXVL+faWlx3I+OjoceeaJW+zJkx0Hx3QP2jzfeervKa/mLF0f+4sW1rtEPjjg0hu06NOn1A6CZ/q1WAgAAYGPlZRUx6bFlDXb/CheX1vt7rgvvZr28JHrlfBm9cuZGRnpp0mtR+sWMJAyQ9eHdnEGjYsqeR0dB+5yGN05K6v89l1U+8640+nV+L/p3nhRpqWUNrjariuv//G6M02befvvtcc4559Q6ZJo8eXIcddRR8dprr8VBBx0UH3/8ca3fKzc3N4466qiYOHFi5OTkBGwrnTp1jKuvuCyuuf7GWgdMG0pLS4uTjv9hjD7q8M32Pees02NhXn7Mmj2n1tvPzMyMs888NTp17LhF+3XyT06IvEWLYtLkqVtdm31H7h0/PvFHDaZ+ADQ/QjsAAKCKivKIue8UKEQ1ysoyYu78HSJ3Xo/oXvxOdC9+J9JjTfMsRnl5bDflneg99b34oMfIeL7/6MjPFjZERCxfkxLjc/eNT+fvGG2zH4s2rZ6PlJQShYnGE97NnDkzzj333C2+K2zcuHHRvn37rbqbLD8/Py688MJ46KGHGs3xNM1l/cnKyoqiorr5e9OmTev47z9eEq+/MTYefuzJWLly1WbXSUlJiZ132jFOOv7YGDhgh1q9T8uWLeOqK/4Qjz7+VLz40muxZu3aTW7/gP1HxYnHHxudOnaIuV9+tUWfqXXr7Lj8DxfH8y++Eg8+/Hi109HWZId+28dpJ58UgwcNbFD1o3rt2rZVBKDJSqkwJwEAALCRspKKePHCXIWohbSKIuHdunGTkiq8q2mcpC4R3tUgO6NtHNj7xHj06jfijtvv3Gz/HXfcMSZPnlyn+7B8+fLo0KHDpvczOztOOumkGD58eLRr1y4mTpwYDz/8cMyaNWuz28/IyIgf/ehHsdtuu0WvXr0iNzc37r333mo/R4sWLSI/Pz9at25d5bUpU6bEkCFDEtq6du0aeXl5tf6sY8aMiSuuuCKh7eqrr45LLrmk2v7t2rWLlStXJrQVFRVFixYtIiJi+vTp8corr0TE11Nl3nXXXQl9c3Jy4rLLLktoO/fccyMl5dtPH9ulS5dYXItpD8eMGROXX365k60Ga4uL47PPJsW4T/4T8+bNj2XLl0dhUVG0bdMmOnXsGJ06dYjevXrGviP3jk6dOm71+xQXF8eETyfGFzNnx/IVK2PlqpXRIisrenTvHj16dIsd+m0f3bvVzd+OgoKCmDJ1enw+eWpMnjItFi9ZEmuK1kRxSUm0b9cucrp2iZycrpGT0yW279sn9hy++1aPyfqqHwDNgzvtAAAAvoWylJaRm3VwLMjcr9mHd2kV5TFq3rux9/z3hXcbj5PyTrFs1dmxsuA44d1G1t1593He7Aa7j/vvv3/cc8890a9fv8q2E088Mc4///w46KCD4vPPP69x3b322ivuvPPOGDo08RlZ//Vf/xUXXnhh3HzzzQnta9asiRdffDGOO+64RnH8Bg4cGAMHfn13UnWhXYcOHeK8884z0BuwrMzM2HP4brHn8N226ftkZmbGiD33iBF77rHNP1N2dnYM32O3GL7Hbk2mfgA0D6lKAAAA8O2tC+/Gt740vsr8fpRGi2Zbi3Xh3ZVvXxynfnpHdCnIM0DWjZNvwrt5+ffGyoKjo6IiQ1G+UVpe3CD368ADD4y33norIbBbJycnJ+68s+a7AwcMGBBjx46tEthFRKSmpsZf/vKX6N+/f5XXJk6caEAAADRDQjsAAIA6tGF4tyxtULOuxYbh3WFfPGtwbDhONgjviks826ihSk1NjRtuuGGT0+aNGDEiOnasfsq7m2++ObKysmpcNzMzM0aOHFmlPT8/X/EBAJoh02MCAADUsTals6J38cvRrmxWs6/FolZd47n+o+ODnqMMjAQV0TLrg2jf+oHIzDBOGqqTTz45hg0btsk+qamp0bNnz1i6dGlC+4gRI+KQQw7Z7Husm1pyQ0I7AIDmSWgHAABQR4R1620Y1lWkmORlPWFdY3LooYfWql+HDh2qtA0ePLhW67Zu3bpKW2FhoeIDADRDQjsAAIBvSVi3nrCuJsK6TRnccc9Y3CUnPouHG9R+9enTp1b9WrSo+gzL6p6BBwAAmyK0AwAA2ErCuvWEdTUR1m3K4I57xqHbnRp92+4Yn2ae2+D2r2/fvlu9rtAOAIAtJbQDAACqSEmJaN2t4f7nQuGS0igvSd77t03LjT6ZH0T79NxvWnombV/KV66IitWrk/b+S1t3jbd3Gh2f9R0VFampSaxEVfNXRZRXJOvdK6J9q0+jZ4dnIztr3Tjp22Bqs7hoXpRVlCbt/TcM6xqqjIyM6Nat21av36pVK39MAADYIkI7AACgitT0lDjgsu4Ndv/e/UteLJ9dXO/v27F/Vgw8om10Htg7IvZpELUoeORfseb5/6v/MdI1J1qO/mF0HLVfDEhtmHfWnf50cSxfU//vu2fPlDhx5/TYvsOIiBjRIGtz9YcnR35Rbr2/b2MI69bp0KFDpKY2vbtGi4uLAwCAhkloBwAAsBnrw7oWzb4W68K6rFH7RUqqaTA39HVYlxbbd1CXjTWmsK6pW7ZsmSIAADRQQjsAAIAaCOvWE9bVTFhXM2FdwyO0AwBouIR2AAAAGxHWrSesq5mwrmbCuoYrLy9PEQAAGiihHQAAwDeEdesJ62omrKuZsK7+lZWV1bpvcXFxjBs3TtEAABoooR0AANDsCevWE9bVTFhXM2Fd/UhJSanStmTJkli8eHF07tx5s+t/8MEHUVhYqJAAAA2U0A4AAGi2hHXrCetqJqyrmbCufnXq1Kna9vHjx8fBBx+82fXvu+8+RQQAaMCEdgAAQLMjrFtPWFczYV3NhHXJ0alTp0hPT4/S0tKE9k8++WSzod2ECRPi3nvvVUQAgAZMaAcAADQbwrr1hHU127NnSpywc1r0E9ZVIaxL8nmbmhrdu3ePr776KqH92muvjcMOOyyGDRtW7XrTp0+P0aNHR3l5+Tbdv+qm75w3b16Ul5dHqusMAMBmCe0AAIAmT1i3nrCuZsK6mgnrGo6jjjoqbr311oS25cuXx8EHHxx33XVX7LvvvtGxY8eI+Pp5d3fccUdcf/31sWTJkm2+b127dq3StmrVqrjkkkvi7LPPjr59+8aaNWuiVatWDiQAQDWEdgAAQJMlrFtPWFczYV3NhHUNz09+8pMqoV1ERH5+fowePTpSUlKiT58+UVJSEgsWLIiKiop627ecnJzIysqKtWvXJrRfe+21ce2111Yul5SURHq6n6QAADbmGxIAANDkCOvWE9bVTFhXM2FdwzVy5Mg4/vjj49FHH6329YqKipg7d261r2VmZsaYMWPi//2//7dN9i0lJSUOPvjgePbZZx0oaIS+mDk75i9YULm878i9t3pq2ylTp0f+4sWV14b9Ru3TIPY/GfvV3Kk5bBmhHQAA0GQI69YT1tVMWFczYV3jcNddd8XMmTPjk08+qfU6rVq1ijvuuCN69eq1Tfftd7/7XTz33HP1eocfbMrYd96PmbNmR0TEbrsOjd2G7aIoNXjk8afiPxM+q1zeZ+8RWx3a3XP/v2LW7DkRUX9BTW32Pxn71RC89sbYePvd9yMiIi0tLcb8v9/V23s315rD1hLaAQAAjZ6wbj1hXc2EdTVr7mFdenp6ZGZmRnFxcYPYny5dumzy9TZt2sT7778fl19+edx8882xevXqmq8JqalxyimnxJVXXhk9e/aMiRMnbtG+ZGdnx8qVK2vdf999942HHnoozj777Fi+fLmTi6QqLy+PB/71SCxfvuKb8dxKaPctFRUVxVPPPB+lZWUREbFDv+1i1D57KUwDNmfuV3HXPQ9EaWnp13/z0tKMBWjI30uVAAAAaKyEdesJ62omrKuZO+u+1rp16yrPYdsSL7/88lave8EFF8QFF1ywxetlZmbG1VdfHX/84x/jpZdeinHjxkVeXl7k5+dHdnZ2DBo0KAYNGhTDhw+P/v37V643dOjQLboLbv78+Vu8byeccEIce+yx8eGHH8b06dNjzZo1kZ2dHZ07d45hw4Z5nh315vNJUyoDO+rGipWr4slnnqtc/s5+IwU1Ddja4uL42823VQZ2xgI0fL4lAQAAjU7HHbJi8A/aCesiIq1L18g+8xxhXTWG90iNQwekCuuqMajj8PhJziWmwWwCWrRoEUcffXQcffTRDWq/0tPTY9SoUTFq1CgHiaRYsXJl3HH3/QqRJN275URZ2ddBUWoD+n7SUPdrW7nvgYcid958NYdGRGgHAAA0OkOOaa8I32jx3YMUoQbnjvCfvDU5dsD5igA0SatWr45/fzAunn/xlVi4ME9BkuTX559jv5Lso3GfxCuvvanm0Mj4LxgAAAAAoNF6c+y78e77H8T8+Qtj8ZIlWzQFLDRFS5Yui1vvuFshoEBh3XMAACAASURBVBES2gEAAABAA7Z27drKZ1KlpaVHixZZm12noLAw4pvwKisra5PPEiwuLo5Zs+fEgoWLYmFeXpQUl0TPnj2iT+9e0atXj2jZYuumo87LWxSzZs+NefMXRHZ2q+jSpXMMGTwoWrVqWaf1mTR5Snz62ef1flxKSkuj+JvnYaakpFZ+roqKipg244uYN29+LF22PNq3axfdu+XEjoMHRlpaWtVjVVAQ02bMjNlz5kanjh1jQP9+0aN7t0hJSUnamKjpsxYUFCa0l5aWRUFBQUREpKalbfVYWadozZqY+PnkyMvLj5WrVkbPHj1iwA7bR/fu3ZI6teK3GculpaWVz01NTU2Nli2/XqekpCSmTf8iZs/5MopLiuM7+42Mzp06fet9LS8vj5v+fnusXv31ccnIyIiysrIoLy+v03Ffl2MhmefStr5+bavrK02X0A4AAAAAGrDfXXp5zJu/ICIi2rdvF3fdduMm+3/08fi49vqbKpcv+OUvYr9R+1TpV15eHmPfeS8eevTJWLp0WY3bG7Hn7nHeL86I7Ozsze5rQWFh/O89/4yPx0+IwsLCKq9nZmTEiD13jxOO+2F075bTqI/LmCv+FNNnzIyIiA4d2sedt/4tJnz2edz/z4fjy69yq/Tv1i0nzjjlJ7HbsF0iIuLDjz6Jhx97Ir7KrfrMsVatWsUPRx8RR//giHodEzX507V/jc8mTqrS/t6/P4z3/v1hREQMHjQg/ufy/xcREZf+8cqYNXtuRER07Nghbr3xusp1ioqK4rSzflW5fPyxR8fRPzg8HnvymXjhpVerhEEREa2zs+PMM06OUfvs9a2O2ab2a1uN5etvvDXGfTw+IiJyunaJv994XUyfMTP+/o+7Ko9hRET/ftvXSWj39P+9EJMmT61c/tmPj4/X33w75n75VZ2M+7ocCw3hXNpW169tcX2leRDaAQAAAEADVlxcUu2/12Td3S6bUlZeHn+69q8x4dOJm+370bjxMWfOl/GbC86L/jtsX2O/mbPmxPU3/j0WLcqv+bOUlMS7738YH40bH6f87KQ45Pvf/db1+c5+o6L/Dv2qtH/2+aT4aNz4bXZcSkvLEpY/+HBc/PXmf0RZWVm1/RcuzIvrbrg5/nzVmHj/g3Hx+JPP1LjtwsLC+OdDj0VaWnocdcQh9TIm6nL90tKyyjsB8/MXJ7xWURGVr0VELMrPj5v+fnu8+/6HNW5vdUFB/PWm22Li51PitFN+HFmZmVt9zGrar201ljf8rBEREz+fHFddc0OV9row44tZ8fBjT1YuD9t1aBx2yEHx+ptv19l71OVYaAjn0rY45tvi+krzIbQDAAAAgGbmXw8/XuUH5Y4d2kdOTtcoLi6OvLz8WL3Bj/OL8hfHf191Tfz9b9dF27Ztqmzvo3GfxA033hqlG/3InpqSEtmts2PVqtUJ7cUlJXHn3fdHenp6fO/A/b/VZxm685AYuvOQKu3FxcXbNLTb0LJly+OGm27b7BSExSUl8btLxlSpU03u++dD0bt3zxi2y85Ndiy+9sbYhOX0tLTo2KljFBYWVk7xuL7vWxERcfaZp26z/dmWY3nxkqVxzfU3bZPArqioKP528/ox2KZN6zjv7DO2amrIZErGuVTXx7yur680L0I7AAAAAGhG1q5dGy+89GrlckZGRpx/7lmx917DE37gn/DZ53HH/95XeedJUdGaePrZF+Lkn5xQZXt33fvPhB+8O3fqGGedcUrsNGRwZGVlxdKly2L6jC/i3gceisVLllb2u+ueB2LvvYZHdqtWjb6u60KGfUfuFceMPjJ69eoZy5Yti6eeeT5efvWNyn4b1ql3rx7x05OOj4ED+8eqVatj3Mfj4/kXX4mly5ZX9nnn3feTHtpd/oeLo7y8PPIW5cfvLr28sn3EnrvHuWedERERaWnf7plzLbKy4icnHRcHffc7kZGRERERX8ycHQ88+EhMmrJ+usc33hwbhx96UPTp3WubnBvbciyXlZVV3j2WnpYWOw0ZHH379on27drGgP47fKt9v/PuByJvg7vEzjnr9OjQvn2jHAv1eS7V9TGv6+srzU+qEgAAAABA8zHx88lRUrJ+SsUjDz849tl7zyp35AzbZef4/W8vSGj7fNKUKtt75tkXE57ZNHDADnH9NVfG7rvtGllZWRHx9bOs9t5rz/jvP14SnTuvf25XSUlJvPveB02mtqed/OO48FfnRN8+vSMtNTU6d+oUZ55+cuwydKcqfXfeace47k9XxB67D4s2rVtHj+7dYvRRh8eFvzonod+UqdOT/rlatmwZ2dnZ0WqjQKplixbRunV2tG6dHS1bttzq7aelpcWlF/8mDjvkoMrALiKi/w7bxx8u/W1C/corKuK+Bx7aJp+zvsZyTtcucdUVl8Vll14UJ//khPjBkYdFq1ZbX7+3330/3n73/crlg757QIwYvnujHAv1fS7V9TGv6+srzY/QDgAAAACakQUL8xKWU1Jq/omwT+9eCc9ZWpi3KOH1ZcuWx9PPvlC5nJ6WFuf94ozIzs6udns5XbvE735zfkLbm2PfaRJ13X67PnHYIQdV+9oeu+2asJyelhZnnn5ypKdXnQhtyI6DosU3YUFExIqVK5v8mPzuAfvFkB0HVftaRnp6nHn6yZGWllbZ9unESZt8Jt3WqK+xnJmREZdf9vvYod92dbLfeXmL4s67769c7t4tJ0792UnOpVqcS9vimNfl9ZXmyfSYAAAAANCMtG/fLmH5pZdfi912HRo7Dh5Ybf+rr7gsKioqqn1t4qTJUVxcXLl8wHf2jZ49e2zy/ftt3zd26Ld9zJw1OyK+ngIxL29R5OR0bdR1PWb0kZGamlqrmvfrt1307NG9xm117twpcufNj4iItWuLo6S0NDLSm+5PuYcf+v1Nvt69W04MGtg/Jk+ZVtm2IC8vunTpXGf7UF9j+agjD40uG9yt9W2UlZXF3265PYqK1kTE13csXvDLs6NFiyznUi3OpW1xzOvy+krz5E47AAAAAGhGdtpxcMJyYVFRXPbfV8dll18dr70xNpZs8MymiIjU1NRIS0ur/GdD8xck3lVS2+dyDRrYP2F53oKFjb6uPbrXHBxseLdPRES3zQSUG0+lt+4ZX01R27Ztonevnpvtt3GfhQvr9q6k+hrLu+06tM72+ZHHn44ZX8ysXD7+2NEJd245lzZ9Lm2LY16X11eaJ3faAQAAAEAz0rFjh/juAfvHG2+9ndA+Zdr0mDLt62c+de3aJXbZeafYfdjQGLrzkBqfUbVgo4DigQcfiRdeenWz+zB/o/WWLVve6OvapXPHWvft3LmzgfiNHbbfrlb9qoR2dTyVYH2N5V49e9bJ/k6aPDWeeua5yuUdBw2MY0Yf2STGRH2dS9vimNfl9ZXmSWgHAAAAAE1Iefnmp1o7+8xTIy0tNV59/a1qX1+0KD9ee+OteO2NtyItLS1G7j0iRh91eGzXt3dCv/kLFiQsry4oiNUFBVu8zyUlJY265llZmTU+B6s6GelpDW5MJEubNq1r1a/XRtMWLl26rE73o77GclYdTF25enVB3Pj32yunVWzZskX86rwza5xS0rlUv8e8rq6vNE9COwAAAABoQoqKijbbJzU1NX7x81PjkO9/L55/8eX4ePyEWLVqdbV9y8rK4p33/h3//uCj+MOlv42dh+xY+drKlavrZJ8b+/SPaWnpjX5MJEtmZmbtOibOchgZGXVb88Y0lmfNmZMQWhYVrYl/3Hlvjf3nfvlV5b+XlpXFFVdfV7ncuVOnOPcXpzfLc2lbHfO6ur7SPAntAAAAAKAJWV1QWOu+2/XtHeed/fMoLy+PL2bOjkmTp8Tnk6fGlKnTo7i4OKFvaVlZXPuXm+LG6/8UHTq0j4iIrl06x5Kl65/R9PPTfhZ9+vTa4n3u3297B66BjIn6lpeXX6t+S5Yk3llX11MKNvax/NnESVvVt2eP7s32vNjWx/zbXl9pnoR2AAAAANCE5G3Fs75SU1Nj4IAdYuCAHeKY0UdGSUlJTJo8Nd546+14/4Nxlf0Ki4pi8tTpMWqfERER0b17t8rnNEVEtG/fLoYMHuQgNIExUV/mfvVVrfotWbI0YXnj6TK/LWO5+amvY76111eaJ6EdAAAAADRgKRtMC1hUVBRr1qyNFpt4Ltbnk6dscnv3/fPhWLt2bURE9O3TOw75/ner9MnIyIhhuw6NYbsOje7dn4gnnnq28rWZs2ZX/qjcs0e3hPVmz5kbe48YvtnP9NobY2PW7DmV7/XjE46NrKwsBztJYyKZVq5cFStWrox2bdtust8n4yckLA/dqW6nEWxMY7lTx47Vnrc1+XDcJ7F8+YrK5Q3Xbd++XbM9j7bFMa/L6yvNk9AOAAAAABqw7OzsiPzFERFRUVERc+bOjcGDBlbb962334tly5ZvcnsffPhx5C9e/M22W8X3Dtw/0tNr/plwxPA9En5U3nBatx7dE6fWe/W1t+KHo4/cZGiRn7847rrngSgtLY2IiNats+NnPz7egU7imEi2Dz78eJMh1OQp02LajC8qlzt27BDdu3er031oTGO5Z4/ucebpJ9e6/5df5laGdulpaVu0blO2LY55XV5faZ5SlQAAAAAAGq6N74T54KNPqu03Z+5Xccf/3rfZ7e2ww3aV/15QUBifT9r0XVjr7ihZp9/269ffbdjQ6LHBM7FWrloVzzz7Yo3bKi8vj/sffKTyB++IiJF7j9jkj9ps+zGxtVauWl0n27nn/n/FFzNnVftaQWFhPPCvRxLa6vouO2O54YyF+rQtjnldXl9pnoR2AAAAANCADR7YP2H5uRdejiee+r9YuXJVRETMX7AwXnrl9bj2+htrdZfGTjsOTli+6e93xLiP/1Ol39ri4njv3x/FPff/K6F9QP9+lf+enp4ePz/1pwmvP/rE03Hn3fdHyQY/bEdE5C9eEmOu/HP8+8P1z3DKzMiIww45yEFO8piorZYtWiQs/2fCZ/H55ClRWloa5eXlW73d0tLSuO6GW2L8fz6NojVrKtu/yp0Xl152Zcz4YlbCmDnqiMPqvKbGcsMYC/VpWxzzury+0jz5X1gAAAAAoAHbd9Q+8dgTz0RpWVll20OPPhkPPfpktGzZIoqK1mzR9r7/vQPildfeiK9y50fE13eXXHP9jdG3T+/o2aN7tGvXNmbPmRtffDEr4T3Xrdu7V8+Etl2G7hR77zU8Pvjw48q2l199I15/8+3o07tXpKSkRGFhYSzMWxQVFRUJ6552yk+qbI/6HxO11a5d28jKyoy1a9cHgZdfeU1EROw5fPe4+L/O3+ptL1m6NK6+9q+RlpYW2/XtHcuXr4wlS5dW6XfGaT+L7fr23iafz1huGGOhPtX1Ma/r6yvNjzvtAAAAAKABy+naJS7+7QWRmZFR5bWNw5lWLVvGr88/Z5PbS09Pj1+ceVqVafzmfvlVvP/BR/Hiy6/F1Gkzqvyg3LtXjzj1ZydVu81zzzoj9t93ZEJbaWlpzJo9J2bOmh0LFuZV+cH7qCMOje9/7wAHuAGMiS2x36h9qm1ftWrVVm2vT+9e0aljx8rlsrKymDlrTrWB3YHf2Te+d+D+27S2xnLyxkKy1OUx3xbXV5oXoR0AAAAANHC7DdslLv39b6Jjh/Y19tl35N5x/TVXRr/t+m52e4MHDohrrhoTffts/o6lrKzMOOFHR8efr7o8srKyqu3TqlXLOP+8s+LX558TrVq12uT2enTvFhf/1wVxyk9PbNTHJD09bavXbbHR1IINYUxsGDJkZmREamr1Px2fdPyxMWjggDqrTfduOXHl5ZfE9tv1qbFPhw7t47yzz4hzf3HGt9r/2uxXXY/l2ta1vtTF2KvrsZDsc6muj3ldX19pXlIqNo6IAQAAAOrZueeeG7fddttm+/Xt2zfuvfdeBSOOPvroWLFixWb7jRkzJi6//PIm9dmXLVseM2fPiTlzvoy0tNTo2aN79O3TO3Jyum7xtsrLy2PO3C9j0uSpMXPW7FhdUBilpaXRuVPHyOnaJXJyusbOQ3aMjh071Hqba9asjTlz58as2V//s2btmujYoUN06tQxhgwe5JlNDXxM1NbCvEUxf/6CqKioiOzW2dG7V8/I3kzgERFRWFgUJ5+x/s6/vfbcIy76za8iImLS5Kkxeeq0WLx4SbRp3Tqys1tFr549Y+jOQ6JFi/oPNIzlbTsWGqK6PObb4vpK0ye0AwAAAJKutqEdbKmmGNpBY7ap0A6guTM9JgAAAAAAACSZ0A4AAAAAAACSTGgHAAAAAAAASSa0AwAAAAAAgCQT2gEAAAAAAECSpSsBAAAAAAD1ITU1NXr36lm5PKB/P0UB+IbQDgAAAACAetGiRVb89bqrFAKgGqbHBAAAAAAAgCQT2gEAAAAAAECSpVRUVFQoAwAAAJBM8+fPj6VLlyoEda5r167RtWtXhQAAGjyhHQAAAAAAACSZ6TEBAAAAAAAgyYR2AAAAAAAAkGRCOwAAAAAAAEgyoR0AAAAAAAAkmdAOAAAAAAAAkkxoBwAAAAAAAEkmtAMAAAAAAIAkE9oBAAAAAABAkgntAAAAAAAAIMmEdgAAAAAAAJBkQjsAAAAAAABIMqEdAAAAAAAAJJnQDgAAAAAAAJJMaAcAAAAAAABJJrQDAAAAAACAJBPaAQAAAAAAQJIJ7QAAAAAAACDJhHYAAAAAAACQZEI7AAAAAAAASDKhHQAAAAAAACSZ0A4AAAAAAACSTGgHAAAAAAAASSa0AwAAAAAAgCQT2gEAAAAAAECSCe0AAAAAAAAgyYR2AAAAAAAAkGRCOwAAAAAAAEgyoR0AAAAAAAAkmdAOAAAAAAAAkkxoBwAAAAAAAEkmtAMAAAAAAIAkE9oBAAAAAABAkgntAAAAAAAAIMmEdgAAAAAAAJBkQjsAAAAAAABIMqEdAAAAAAAAJJnQDgAAAAAAAJJMaAcAAAAAAABJJrQDAAAAAACAJBPaAQAAAAAAQJIJ7QAAAAAAACDJhHYAAAAAAACQZEI7AAAAAAAASDKhHQAAAAAAACSZ0A4AAAAAAACSTGgHAAAAAAAASSa0AwAAAAAAgCQT2gEAAAAAAECSCe0AAAAAAAAgyYR2AAAAAAAAkGRCOwAAAAAAAEgyoR0AAAAAAAAkmdAOAAAAAAAAkkxoBwAAAAAAAEkmtAMAAAAAAIAkE9oBAAAAAABAkgntAAAAAAAAIMmEdgAAAAAAAJBkQjsAAAAAAABIMqEdAAAAAAAAJJnQDgAAAAAAAJJMaAcAAAAAAABJJrQDAAAAAACAJBPaAQAAAAAAQJIJ7QAAAAAAACDJhHYAAAAAAACQZEI7AAAAAAAASDKhHQAAAAAAACSZ0A4AAAAAAACSTGgHAAAAAAAASSa0AwAAAAAAgCQT2gEAAAAAAECSCe0AAAAAAAAgyYR2AAAAAAAAkGRCOwAAAAAAAEgyoR0AAAAAAAAkmdAOAAAAAAAAkkxoBwAAAAAAAEkmtAMAAAAAAIAkE9oBAAAAAABAkgntAAAAAAAAIMmEdgAAAAAAAJBk6UoAAAAAJNvrr78eEyZMUAjq3MiRI2OfffZRCACgwRPaAQAAAEn3xBNPxG233aYQ1LkxY8YI7QCARsH0mAAAAAAAAJBkQjsAAAAAAABIMqEdAAAAAAAAJJnQDgAAAAAAAJJMaAcAAAAAAABJJrQDAAAAAACAJBPaAQAAAAAAQJIJ7QAAAAAAACDJ0pUAAAAAaCxSUlIiIyNDIYji4mJFAACaFKEdAAAA0GgMHjw4Jk+erBBEly5dYvHixQoBADQZpscEAAAAAACAJBPaAQAAAAAAQJIJ7QAAAAAAACDJhHYAAAAAAACQZEI7AAAAAAAASDKhHQAAAAAAACSZ0A4AAAAAAACSTGgHAAAAAAAASSa0AwAAAAAAgCQT2gEAAAAAAECSCe0AAAAAAAAgyYR2AAAAAAAAkGRCOwAAAAAAAEgyoR0AAAAAAAAkmdAOAAAAAAAAkkxoBwAAAAAAAEkmtAMAAAAAAIAkE9oBAAAAAABAkgntAAAAAAAAIMmEdgAAAAAAAJBkQjsAAAAAAABIMqEdAAAAAAAAJJnQDgAAgCappKxCEWpQWl6sCAAA0MAI7QAAAGiSLnqlNJ6eUhZrSoV3G7v5PxfGK3MeiDWlBYoBAAANhNAOAACAJmnl2oq4/9OyOPvZEuHdRopKV8eLc+6JKz44SXgHAAANhNAOAACAJm3l2hDe1UB4BwAADYfQDgAAgGZBeFcz4R0AACRfuhIAAADQnKwL756eWhZHD06LQwekRov0FIWJ9eHdW7mPxQG9jov9e/0wWqRnK0wz9sADD0Rubm5C20UXXRTp6X5SAgCoa75hAQAA0CwJ72omvGOdX/7yl7Fy5cqEtl//+tdCOwCAbcA3LAAAAJo14V3NhHdsifLy8li9enVCW2pqarRu3VpxAABqwTPtAAAAIDzzblM8847amDhxYrRr1y7hnz333FNhAABqSWgHAAAAGxDe1Ux4BwAA247QDgAAAKohvKuZ8A4AAOqe0A4AAAA2QXhXM+EdAADUHaEdAAAA1ILwrmbCOwAA+PaEdgAAALAFhHc1E94BAMDWS1cCAAAA2HLrwrunp5bF0YPT4tABqdEiPUVhYn1491buY3FAr+Ni/14/jBbp2Y3+c1VUVERubm7MmDEjli1bFr169Yp+/fpFl//P3n2HN1W9cQD/JumiA7pL6S7QlpYNQilLpEzZU0EoW5SfKKigDFmCgLJdIAJOBBmyK1CggIyWDR3M0tK990jS9PeHWglpk3Qn6ffzPD6Se8+9uXnPOenNfe89x8ZG5bY5OTmIiopCbGwsCgoKYGtrC3t7e7i7u0Mo5D3VRERERMSkHREREREREVGVMHlXPm1J3uXm5mLQoEFyy/r27YsFCxYAANLS0rBx40Z8+eWXyMzMVNje0dERixYtwpQpU6Cvry+37vjx49i6dStOnDgBiURS5ravv/465s+fDysrK6XHGR0djYCAALllnp6e2Lp1q9qfddeuXdi1a1fpa6FQiHXr1qFdu3aVit2HH36I0NDQ0ji+KCYmBi+//LLcsqCgIIhEInYQIiIiohcwaUdERERERESVUiwrQUah5h6frJZHrXw+eTfUSwg/pwwYauiv7uKS4lp9P01P3kmlUgQHB8sti42NxYIFC7Bv3z5Mnjy5zITU82VnzpyJzZs348yZM7Czs0NWVhZmzZqFX375Rel7x8bG4vPPP8fu3buxb98+dO7cudyy+fn5CscZERFRoaRdVFSUwj4CAwMrnbS7ffu2wv5UHXNJCYeUJSIiIioLk3a1LDk5GTExMYiNjUVaWhrS09ORnp6OwsJCiMViiMViAICBgQEMDQ3RoEEDWFpawtLSEtbW1nB0dISLiwssLS0ZTCIiIi2Vm5tbeh6QnZ1deg4gkUigr68PAwMDGBgYoGHDhrCysoKlpSVMTU0ZOCLSOPE5wLsnJAzEC7KLgJ9uy/DLnRI0NNkHM+MjEAqLGBho37CZW7duxVtvvaV2kik8PByDBw/G6dOn4e/vj2vXrqn9XrGxsRg8eDDu3r0LOzs7NhYiIiKieohJuxqSnJyMq1ev4vbt2wgLC0NYWBgePXqEgoKCatm/qakpPDw84OPjg5YtW6JNmzbo3LkzzM3NGXwiIiINkJSUVHoOEB4ejqioKERHRyMmJgb5+fkV3p+xsTFcXFzg7OwMNzc3eHt7l54H2NraMuBERBpIVmKOzNxpyM4bxeTdC8pK3mmax48f4+23367wU2GhoaEwNzev1NNkKSkpeO+997B7926tqUsOc0lERERUfZi0qyaxsbE4deoUgoKCcOnSJURFRdXo++Xm5uLGjRu4ceNG6TKBQABPT0/4+fnB398f/v7+ak2GTURERFUjk8lw8+ZNBAcH48qVK7hy5QqePXtWre+Rn5+PiIgIREREKKxzdnaGr68vfH190aNHD7Rr1w5CoZAVQ0SkKX8nmLwr1/PJu2fZeRr5N/5fJiYmeP3119GxY0c0atQId+/exW+//YYnT54obPdiwk5fXx+jRo1Cu3bt4OjoiNjYWOzatQvh4eEK2/7xxx/Izc3VmqfsN23ahJMnTwL4e6jM7du3y623s7PD4sWL5ZYx0UdERERUNibtqiAkJAQHDhzAoUOHEBkZWefHU1JSgsjISERGRmLHjh0QCARo27Ythg0bhhEjRqBly5asNCIiomqSnp6Ow4cP48SJEwgKCkJaWlqdHUtMTAxiYmKwd+9eAICVlRX8/f0xYMAADB48mMNqExFpCCbvylcgzUVMzn2NPb4ePXpg586dcHd3L1322muvYfbs2fD398e9e/fK3bZz58747rvv0KpVK7nl77//Pt577z1s2bJFbnlhYSFOnDiB0aNHa0XdeXh4wMPDA0DZSTsLCwvMmjWLjZyIiIhIDUzaVVBERAR27dqF3bt3V/sd9NWtpKQEN2/exM2bN7FkyRJ4eHhg/PjxCAgIgIuLCyuTiIiogjIzM7F37178/vvvOHfuHKRSqUYeZ1paGvbs2YM9e/ZAT08PvXr1wujRozFmzBg0atSIFUlEVMeYvNMuvXr1QlBQEAQCgcI6Ozs7fPfdd+jSpUuZ2zZv3hzBwcEwNDRUWCcUCvHFF1/gxIkTePTokdy6u3fvak3SjoiIiIiqD8dNUkNhYSF27NgBX19feHt7Y+3atRqfsCvLgwcPsGTJEri5ucHf3x/79u1DcXExK5iIiEiJkpISnD59GuPGjYO9vT3efPNNnD59WmMTdi+SSqU4deoUZsyYAXt7e4wfPx5BQUGsWCIiDSAraQSx1APFMmsGQ0MJhUKsX7++zITdvzp16lTuU+1btmwpM2H3LwMDA/j5+SksT0lJYfCJiIiI6iE+aadEYmIivvrqK2zdulWnTphLSkoQFBSEcVydsgAAIABJREFUoKAgODs7Y9asWZgxYwbMzc1Z6URERP/Iz8/Hjz/+iM2bN5c5j5w2KigowK+//opff/0V3t7emD17NiZMmABjY2NWOBFR7f4qg7HRBTQy+QUG+tEMhwabOHEi2rZtq7SMUCiEg4MD0tPT5ZZ36tQJ/fr1U/ke/w4t+Twm7YiovisoLMStW3eRmJyMzMwsZGZmQSyRwNTEGKamprCxtoJ3Cy+4ODsqvbGCiEjbMGlXhvj4eKxZswbbtm1DYWGhTn/WmJgYzJ8/H6tWrcLs2bMxZ84cWFhYsBEQEVG9lZWVhc2bN2Pjxo0KF990SXh4OGbOnIkFCxZgzpw5mD17Nho2bMgGQERUo5isU8bCqDGAeI06pv79+6t37GX8jvby8lJrW1NTU4Vl+fn5bBBEVC9FRD7A0RMncfPmbYglEpXlGzVqiEED+2FAX38YGRkygESk9Zi0e056ejqWL1+Ob7/9FkVFNT+fgKGhISwtLWFlZQVTU1MYGBjAwMAAJSUlEIvFEIvFyM7ORnp6OtLT0yFR4w9VZWVlZWHFihXYuHEj3n//fXzwwQcwMTFhoyAionojNzcX69atw8aNG5GZmVmvzn8WL16M9evX47333sPcuXPLvHhIRERVwWSdMj5WfujvGoBVVhsQjBsadWzOzs5qlTMyMlJY5u7uzsolIlKTVCrFb78fxKEjx1FSUqL2dllZ2fhl9+84EXgKC+bPhauLM4NJRFqNSTsAYrEYW7ZswaefflrtF+n09fXRsmVLtGzZEj4+PvD09ISLiwtcXFzKHfO+PMnJyYiJicHTp08RGRmJsLAw3Lt3D+Hh4ZDJZNVyvDk5OVi6dCm2bduGFStWYNKkSRAKOfUhERHpruLiYnz//ff45JNPkJSUVO37FwqFcHd3h4+PD9zd3eHs7AxnZ2fY2NjAysoKFhYWMDIygoGBAfT19SGRSCAWi1FYWIiMjAykpaUhJSUFMTExiImJwePHjxEWFoaoqKhq+/sPABkZGViyZAm++eYbrFixApMnT4ZIJGIDISKqEibrlPk3Wedo1lxjj9HFxaXS2zJpR0SkHplMhs8+34jbd+5Veh/pGZlYvGwVliych2ZN+f1LRNqr3iftzp07h5kzZ+L+/fvVsj9jY2P06NEDvXv3RpcuXdChQ4cy77irDFtbW9ja2qJjx45yy3NzcxEaGopLly7h9OnTuHTpEsRicZXeKz4+HlOnTsW3336Lbdu2qRzDn4iISBtdvHgRb731Fu7du1dt+3RwcECPHj3g6+sLX19ftGrVCg0aNFB7ewMDg9Kn3e3t7cstV1BQgDt37uDKlSu4cuUKzp8/j/j4qg8plpiYiOnTp2Pz5s349ttv4efnx4ZCROWyNgbmd9Pcn5VfXpUiT1I37+3rCHRwuAdbUyMAUzUuNvsebkK2OK1O3lsbknXA3zfhNm7cuErXB4iISLW9+w9VKWH332+kQnyzbQfWrlrGGxCJSGvV26RdWloa3n//ffzwww9V3pe9vT2GDx+OESNGoHv37jAwMKjVz2JqaopevXqhV69eWLhwIfLy8nDmzBkcOHAAhw8frtJ8PKGhoejYsSPeffddrFixgj86iIhIJ2RkZGD+/PnYvn17hYZeKYtIJELPnj0xZMgQ9OnTB97e3rXyGRo0aIDOnTujc+fOePfddwEAYWFhOHXqFA4fPozz58+juLi40vu/e/cuunXrhunTp2PNmjUwNzdnwyEixe8ifQE6Owo09vi2XgNQy0k7PychxrQUwrmREEAHjY3NkSfbav09tSVZ9y8LCwudHHmmqjf5EhFVp0ePo7D/4OEy19nZ2qBvn1fQ+aUOsLayhEwmQ3xCIp5ERWPv/j+Qmqp480l0TCxO/Hkagwb2Y3CJSCvVy6RdYGAgpkyZgoSEhErvw9TUFKNGjcLkyZPRvXt3CASa80PVxMQEgwcPxuDBgyGVSnHy5Ens2rULhw8frtRcfcXFxVi/fj2OHDmCn3/+GZ06dWLPISIirXXs2DFMnTq1SkNhCgQC9OjRAxMmTMCwYcNgZWWlEZ/Nx8cHPj4+eO+995Camoo//vgDP/30Ey5cuFCp5GRJSQm2bduGw4cPY8eOHRgwYAAbEBFROeSTdST390nLknW6LiMjg0EgIo1x+kxwmb9VunbpjLdmTIGRkaHcclcXZ7i6OMPPtxN++e13nPjztMK2v/1+EF27dIaFBW88JCLtU69+TRQUFGDWrFkYMGBApRN2LVq0wDfffIOEhATs3LkTPXr00KiE3Yv09PQwcOBA7N27F/Hx8Vi7dm2lx+R/+PAhunbtimXLllXpzn0iIqK6kJeXh5kzZ2LQoEGVTtjZ2Nhg4cKFePToEc6dO4epU6dqTMLuRdbW1pg2bRqCg4Px8OFDLFiwANbW1pXaV2JiIgYOHIi3334b+fn5bExERM/xcxJi4wA9fNBVjwm7F/hY+eH9DlsxrdWnTNhpECbtiEhTiMViXLp8VWG5R/OmmP2/NxUSds8zMjLE1ElvoE0rH4V1hYWFuHQlhAEmIq1Ub560e/z4MUaNGoVbt25VavsePXpgwYIF6NdPex+ttrS0xIcffoi5c+fi4MGDWLlyZYXjIZVKsXTpUgQHB+O3336Dra0texEREWm88PBwjBw5EpGRkZXa3sfHB++//z7GjRsHQ0NDrfv8TZs2xcqVK7F48WL8+uuvWLduHcLDwyu8n2+++Qbnzp3D/v370aJFCzYsIqrX+GSdkr+bfLJOo1VltAHSfMUyGeLiEhCfkAB9PT00a+aORg0bqr19aloaMjKykJOTA0NDQxgaGsDG2hqNGjXU2phkZmYhMSkJCYnJKCwshImJMRqamcHOzhb2je2q9b10MX416fqN28gvKFBYPnP6ZIjUHJ545vTJePeDBQpD/z6NjmGAiUgr1Yuk3ZEjRzBx4kRkZmZWeNsePXpg2bJlePnll3UmHiKRCKNGjcLIkSNx+PBhLF26tMLJu7Nnz6J9+/bYu3cv/Pz82JOIiEhj/fbbb5g2bRry8vIqvG3r1q2xePFijBw5UqOfrFeXkZERpkyZgkmTJmH//v1YsWIF7t69W6F9REREoFOnTvj+++8xZswYNjAiqneYrCsfk3V1oyIj4YjFYoSGhjJoWmrlmvVy05681KE9Br/6983lBQUF+PGXPQg+/xfEkv8m83zn7eno2b2r0v3GPItD0NlghITeQEpqapllrK2t0LO7H17u3hX29o1VHmtKSiq2fPOd3LIm9vaYOX2S2p/3bPAFnA2+WPpaIBAg4I3X4e6megSppKRk/LpnP67fvI3CwsJyy9naWKNjh3YYPuTVSg+lWBPxqy9iYmMVljk7OcLZyVHtfdjYWKNZUzeER9yXW/40+hkDTERaSeeTdmvXrsVHH31U4XlcPDw8sHbtWgwdOlRnYyMQCDB06FAMHjwYP/74IxYtWoS4uDi1t4+Li0OvXr2wfft2TJgwgb2JiIg0ikwmw/z58/HFF19UeFsnJyesWrUK48eP14lk3YuEQiFGjx6NUaNG4eeff8aCBQsQW8YP5vLk5uZi7NixuH79OlavXq2TMSIiehGTdeXzsfJDP9eJcDLzYDBq4Xf8i9LS0pCamqrWMNhXrlzhUNdaLPL+AxQU/JeAysvLx+BX++H+g0dYv+lrpKWnV2h/eXl52L33AE6eOgOZiutmqalp2H/wCPYfPIL+fXtj/Ouj0cDIqNzyRWKxQhIlNi6+Qkm7pORUhX3cun1XadIuLy8Pe/cfwp8ngyBVI6GdnJKK44GnEHQ2GK+PGYlBA/tpRPzqi7Q0xTbbsUO7iv9+c3Qos73JZDIIhfy7TUTaRWe/taRSKWbMmIH58+dXKGFnbGyMtWvXIiwsTKcTdnKNQCjEpEmT8ODBA8yfPx96eurncsViMSZOnIilS5eyNxERkcYoKCjAqFGjKpywMzIywooVK/DgwQO88cYbOp+MEggEmDBhAh48eIDly5fDqIIXDtauXYvRo0ejoIwhbYiIdAXnrCufj5Uf5nb4FtNafcqEXS0pby7dGzduqLX9Dz/8wCDqmKin0Vi5el2FE3bp6RlYuGQlAk8GqUw4vSjwZBDmfLhQ44YflEql+OzzjTh24qRaCbvnFRWJseun3fh9/x/1Nn51oaykXWO7ik/FU9bvmOLi4go/xEFEpAl08hdHQUEBhgwZgu+++65C2w0YMABhYWH48MMPK5S40hXGxsZYvXo1bty4AV9f3wptu2zZMkyZMqVCw3IQERHVhNTUVPTs2RMHDx6s0HZ9+vTBvXv3sGjRogonr7RdgwYNsHjxYty9exf+/v4V2nb//v3o1asX0tLS2PiISKf4OQmxoT+TdWVhsq7uWFlZlXm94vr16yq3vXXrFnbt2sUg6pCc3Fx8unpdmXOCKZOWnoFFS1ciNi6+Cufcafh09Tokp6RoTDx2/PALIu8/LHe9UI0b8vbs+wO3bt+tl/GrCxmZWQrLLC0tKryfmBjFoTAdHZpAJBLxi4KItI7OZaZycnIwePBgBAcHq72NiYkJ1q9fjxkzZrBFAGjVqhUuXryI1atXY9myZZA8Nxa6Mjt37kRubi5++eUX6OvrM5BERFTrEhIS4O/vj/DwcLW3MTU1xcaNGzF16tR6H79mzZrh1KlT+O677zB37lzk5uaqtd3Vq1fRs2dPnD59Go0bc44OItJufk5CjPYRwsWciboXcRjMuicUCmFvb49nz+QvUK9duxYDBgxA27Zty9zuwYMHGDp0KGQyWY0eX1mjFMTFxXGIuhqSnp5R7jqDcq7LyGQybP5qK5JTUsusvx7dusDL0wNOTg7Izy/Ag4eP8fDhI4RHPlC4PpSZmYXP1m7EF2tWQFTH9fskKhonT59VWO7s5IihgweiTSsfNGrUEGKJBElJKYhPSMDBQ0fx+MlThW1+3v072rRuWWZ71tX41ZW3ZkzBo8dP5Ja5uTpXaB95+fl48OixwvKK7oeISFPoVNIuMzMT/fr1Q0hIiNrbdOzYEbt370azZs3YGp4jEomwcOFC9O/fH6+99hoePXqk1na///478vPzsX//fhgaGjKQRERUa54+fQp/f388fvxY7W38/Pzw008/wd3dnQF8zvTp0/HKK6/gjTfewJUrV9TaJiwsDN27d0dQUBCcnfkDmYi0D5N15WOyTrMMHjwYX3/9tdyyzMxM9O3bF9u3b0e3bt1gaWkJ4O/57rZt24Z169bVylPxtraKw9rl5OTg448/xsyZM+Hi4oLCwkIYGxuzImuAl2dzDBrQFy7OTrCzsy0zUXrsxEmEhUcqLLeytMTsWTPg4+0lt7x929YAgIjIB1i5Zj0KCwvl1j+LjcP5C3+hV8/udfrZr9+8pbCsqbsbln/ykdz1KUMDAzg7OcDZyQEvdWyPY8f/xI+/7JH/XREdg7CISLT0blFv4ldXmjdzR/NmVfsttmPXz8jLU5yr09XFhV8KRKSVdCZpl5ubiwEDBlQoYTdjxgxs3ryZySUlOnTogGvXriEgIACHDh1Sa5tjx45h7Nix2LdvX70cZpSIiGpfbGwsevXqhadPn6q9zdy5c7FmzRr+rSpH06ZNcf78ecybNw8bN25Ua5tHjx7h5ZdfxoULF+Dg4MAgEpFWYLKufEzWaabx48crJO0AICUlBUOHDoVAIICzszMkEgkSEhJqdU4nOzs7GBoaoqioSG752rVrsXbt2tLXEomE52DVSCAQYPjQQRg7erjSJ7aKxGIcOHRUYbmNtTU+/2wZTE1Nyt22hZcHFn38PlauXoeCAvnE0/6DR+o+aXdDMWk3cvhgpdf8REIhhgwagIj7DxF6TX5eyJiYWIWknTbETyKRYPNX22ol5nNmv1XnT9Cev3gJwRcuKSzX09NDh/Zt+OVARFpJJ86QCgsLMWTIELXvBNfX18e3336LKVOmsAWooVGjRjh48CBWrVqFxYsXq3XCf+jQIQQEBOCnn37iEBhERFSjkpOT4e/vr3bCzsTEBLt27cKoUaMYPDXOmTZs2IAuXbpgypQpyMvLU7lNVFQU/P39cf78edjY2DCIRKSxmKwrH5N1Gt52/fwwZswY7N27t8z1JSUliI6OLnOdgYEBlixZgoULF9bIsQkEAvTt2xdHjhxhRdWiwa/2x7ixI1WWO3/hEnJyFIc/f3NagNKE07+8PJrj7RlTsW7TV3LLE5OSERefAIcm9nUWg8Sk5LI6g1rb9urZTSFpFxefoJXxKy6W4fLV0FqJ+Xt4q07b/YE/juLXPfvKXDd29HA0seew/USknbQ+aSeTyfDaa6/h7NmzapW3sLDA/v370atXL9Z+BU+8Fy5ciObNmyMgIEDhcf6y/PrrrzA3N8dXX33FABIRUY3IyspCnz59cP/+fbXKN2nSBEePHkW7du0YvAoYM2YMmjZtisGDByMhIUFl+cjISPTt2xfBwcFo2LAhA0hEGoXJuvIxWac9tm/fjsePH+P69etqb2NsbIxt27bB0dGxRo9t3rx5OHr0aK0+4VefmTdqhNEjhqhVNvjCX4rfib4voW2bVmq/X6dOHWBpYY70jEy55XfuhtVp0k4kEiksOxl0Du3atYG+iqc6W3q3wOxZM+SWNbazrVfx0yZJySnY9v0PuH3nXpnrmzdriiGDBjBQRKS1tP5Xyty5c9UettHJyQmXLl1iwq4KxowZg6CgIJibm6tV/uuvv8aGDRsYOCIiqnZSqRSjRo3CnTt31CrfqlUrXL16lQm7SurQoQOuXr2Kli1bqlX+1q1bGD16NKRSKYNHRBrBz0mIDf318EFXPSbsXuBj5Ye5Hb7FtFaf1tuEnZ6eHgwMDDTmeFQ9rW5mZoZLly7h448/hqmpqdKyQqEQkydPxoMHDzB+/PjS+e7UZWJiUqHy3bp1w+7du9W+bkBV8+qAPmjQoIHKchKJBI8eRyks79ypY4XeTyQUlrlN1NPoOo2Dp0czxfPR23fx2doNSEhMUrqtsXED9OjmJ/efR/Nm9Sp+2qC4uBiHjp7AnA8XlpuwMzUxwTtvT1c6TCwRkcafl2rzwX/11VfYtGmTWmWbNm2KoKAguHAS0qr/2PXzw7lz59CnTx+kpKSoLP/BBx/A3d0dQ4cOZfCIiKjavPXWWzh9+rRaZTt37owTJ07AwsKCgasCJycnBAcHo3///ggNVT3szsmTJzFr1ixs3bqVwSOiOiEAn6xTFh0+WfcfU1NThXnYKuLPP/+s9Lbvvvsu3n333QpvZ2BggFWrVuGTTz5BYGAgQkNDkZSUhJSUFJiYmMDT0xOenp7o2LEjmjX7LwHRqlWrCj0FFx8fX+FjGzt2LEaOHImrV6/iwYMHKCwshImJCaytrdG2bVvOZ1eN3FzVu8716HFUmTdTubtW/DqZi7OTwrLklNQ6jYNPCy+EhN5QWH7nbhjenfsRvL290KFdW7T08YKLs1OFp3LRlvjp6Ylga2uD5OQUnWrnt27fxc4ffy1z2NLSunBzxfvvzYKdLYfoJyLtprVnScHBwWqf1Hp5eeHMmTOwt+dj5tWlTZs2OH/+PF555RWVw2TJZDK88cYbCAkJQYsWLRg8IiKqsg0bNmD79u1qle3ZsyeOHj2q8i50Uo+lpSWCgoIwaNAgnD9/XmX5bdu2wdvbu1IXI4mIqmqlvz4amwoYiDLMaP0ZrBs0YSB0gJGREYYNG4Zhw4Zp1HHp6emha9eu6Nq1KyupBjk5OqhVLrmcm66/3rrj7zscKiA2TjGR++Jwj7XNz7cTjhz/E6mpaQrrZCUluBcWgXthEQD+Hiq2hZcHWrf0RutWPmrFUFvip6enhy83rIFMJqvxmNfG02wJCYnY9dNuXL95W2m5/n17I+CN16Cvr88vBSLSelqZtEtISMBrr72G4uJilWXd3Nxw+vRpJuxqgJeXF06dOoWePXsiLS1Nadnc3FyMHDkSISEhvGhKRERVcvHiRcybN0+tsl27dsWxY8cqPKwTKWdmZobjx4+jT58+uHz5ssryH374IV566SX4+fkxeERUq5iwKx8TdkS6cl6m3jWW3Ny8MpdH3H9QLcchFovrNA4WFuZY/slH+GT56jITd8/Lz8/H9Ru3cP3Grb+/D62t0LVLZ/Ts3hXOTg5aHz+hUFjhJwk1jUQqxZ7fD+LosUBIlVz/dXN1xvQpExWGMyUi0mZa9w1eXFyMMWPGIDExUWVZBwcHBAUFwcHBgTVdQ3x8fPDnn3+iYcOGKstGRERg2rRpDBoREVVaYmIixowZo9Y8ae3bt2fCrgaZmJjg+PHjas0RKJFIMHr0aCQnJzNwRERERHUgp5ykU3WRSqp/HuOKzo1sa2ODFZ98jM6dOkIgUP+mjdTUNBw6chxz5y3E5xu+RF5enk7ET1vFxsXj40XL8cfhY+Um7ExMjDF10htYvXIpE3ZEpHO0Lmm3cuVKXLx4UWU5MzMzHDt2DG5ubqzlGtahQwfs27dPrTHp9+zZgx9++IFBIyKiCispKUFAQIDKYZmBv+eyDQwMRKNGjRi4GmRubo7AwEC1zrfi4+MREBDAoBERERHVAZkao1VVhUBY/U82l5U8U8XGxhofzvkfNq9fjUED+1Z4frOrIdewZt1mSCQSrY+fNvrz1BnM+3gJnkbHlLleJBLh1QF98eXGtRjQz79WhugkIqptWjU8ZmhoKFasWKGynEgkwp49e9CmTRvWcC3p06cPvvnmG0yfPl1l2dmzZ6Nnz55wdXVl4IiISG1ffvklTp48qbKchYUFjh07BhsbTkBeG2xtbXH8+HF06dIFmZnK5+IIDAzE119/jbfffpuBIyIiIqpFJqZljz7x4Zx3qmX/TZo0rvZjrkzS7l/2je0wacI4TJowDvEJibhzNwz3wiMQHn4f2Tk5SrcNj7iPb7btwOxZb2p1/LTNnn0H8fv+Q+Wu7/xSB0wYPxaN7WzZoYlIp2lN0q6wsBATJkxQ69H4zz//HAMGDGDt1rJp06YhLCwMGzduVFouOzsbAQEBOHfuXIWGKyAiovorMjIS8+fPV31io6eH/fv3w9PTk0GrRV5eXti3bx/69euncs7hDz/8EL1792YdEREREdUi0zKGjBcIBGjXthUMDAw08pgzs7KrZT9N7BujiX1j9O/bGyUlJYiOicWt23dw49YdhEfcL3Obi5euYub0yaWx0Zb4yWQyxDyLq5X3cnVxqrZ9XfjrcrkJu8Z2tpg2eQLatmnFjkxE9YLWJO1WrlyJ+/fvqyw3evRozJkzhzVbRz7//HOEhobir7/+Ulru/Pnz2L59u1pP5hERUf1WUlKCadOmoaCgQGXZ1atXo1evXgxaHejduzdWrVqlMrman5+P6dOnIzg4mDfvEBEREdUSCwvzMs+zE5OS4ezkWCvHIJPJ1C4rkUrx6HFUtR+DQCCAq4sTXF2cMGzIq0hITMK323YiLCJS4VifRseUzpemCfFTh1gswQcfLa6V99rzy45qGZ7ywcNH+Orb78tc17O7H2ZMDYChoSE7MRHVG1qRtIuIiMDatWtVlvP09MSOHTtYq3XZoPT0sHfvXrRr1w7JyclKy86fPx9Dhw6FrS0fayciovJt375d5c0gADBixAi8//77DFgdmjdvHi5fvow//vhDabkLFy5gx44dmDp1KoNGREREVAuaN3Mvc/n9B48qnHQ6duIkzl+8LLfs9bEj0bZ1y9LXZd2alZubh+zsHDRsaKbyPR4+fAyxWKzW8Rw5FoiLl67KLXvn7elwdGiiclv7xnZY8NFczPt4CeLi5efOfvzkaWnSrrbjV59s+/7HMkdWGzFsMMaNHcnOS0T1jlbM1vnWW2+p/EOtp6eHn3/+GaampqzVOtakSRN89913KstlZGTggw8+YMCIiKhcycnJag2L6eTkhO+//54B0wA7duyAo6PqCxfz5s1DSkoKA0ZERERUC8xMTeHk6KCw/PDRExV6Ai4rKxu79x7A4ydRcv8VFhbKlTM1K/v63JOn0Wq9z7nzFyt0TC8ez8NHj9Xe3tDAAD7eXgrL8/Ly6yx+9cWTqGg8jY5RWP5yj25M2BFRvaXxSbuDBw8iODhYZblPPvkEHTt2ZI1qiCFDhmDatGkqy/3888+4du0aA0ZERGVavHgxMjIylJYRCATYtWsXzM3NGTANYGFhgZ07d6oc+jI9PR1LlixhwIiIiIhqiW9nxetmCYlJuHw1VO19/H7gkEKCybhBA3Ro10ZumZmpKYRlDJ345MlTle8R9TQaZ4PVT9o1dXdTWBZ6/WaFYpOalq6wzNnZsc7iV19c/OuywjI3V2e8NWMyOywR1VsaPTymRCJR6+769u3bY8GCBaxNDbNhwwYEBgYiNja23DIlJSV4//331UrMEhFR/RIWFqbW03OzZ8/GK6+8woBpEH9/f8yaNQtffvml0nLfffcd3nnnHbRo0YJBIyIiIqphA/r649Dh4xBLJHLLd/74K2xtbModAvJfZ4Mv4FTQOYXlvp07Ql9fX26ZUCiEhbk50tLlk2F/HDmOdm1bwc3Vpcz3iE9IxJovNqGkpETtz9WsqWLSLiT0Bm7fuYc2agw5GfU0GmHhEQrLvTya11n8KkukJ0L/vr1rvC05OjSp8nx2JSUluHj5qsLyIYMGQCQSscMSUb2l0Um7rVu34uHDh0rLCIVCbN26lV/mGsjU1BRffvklhg0bprTc+fPncfjwYQwZMoRBIyKiUvPmzUNxcbHSMk5OTvj0008ZLA20atUqHDx4EHFxceWWkUqlmDdvHo4cOcKAEREREdWwhg3NMLB/H/xx5Ljc8szMLCxetgozpgagV89uCiMm5OTmYtv2H8p8oszY2Bgjh5d9Padjh7b489QZuWX5+flYseoLvDVjCry8msPsn2lucnJycSroHA4fO4Hc3LwKfS4bG2s4NLFXmJPu09XrMGzIQIwdNRx6eoqXQEtKSnDj5m1s/nobioqZ9rHkAAAgAElEQVTkp+VxaGKvMPdebcevMvT19DBt8gStaI8xz2KRnq44qsrJ02dx5uyFKu/fwsIcs2fNYMcnIq2jsUm7oqIifPbZZyrLzZo1i8NiarChQ4di6NChOHTokNJyy5YtY9KOiIhKXb58GcePH1dZbsuWLZzPVkOZmZlhy5YtGDFihNJyR48eRUhICDp16sSgEREREdWw18aMQFjEfYU536RSKb7e+j12/vgLXJydYGRkBH09PSQmJSEuPrHcedvefnMK7GxtylzXvVsXhaQdAGTn5GDNuk0AAGtrKxQXFyMjI7NKn+vNaZPwyXL564glJSU4eOgYLv51Fe7urnCwbww7OxuIxRKkpKTiSuh1JCeXPcfy5Inj6jx+ui4xMbnM5RGRD6pl/7b1NK5EpP00dk677du3Iz4+XmkZS0tLLFu2jLWo4datWwcDAwOlZW7cuMG77ImIqNTy5ctVlunfvz+GDh3KYGmw4cOHo0+fPirL8XyOiIiIqHbo6enhg/dmwb6xXZnrCwoKEXn/IW7dvovQ6zfxLDa+3ITTwP594Nup/BvpvTyaw8/3JaXHk5qaVmbCTk8kwutjR6r9ubxbeOKVl3uUuS4lNRVXQ67hwKGj+GbbTny/62ccPhZYbsJuyKv90bZNqzqPn65LTEpihyQiKoNGJu2kUinWrFmjstzixYthYWHBWtRwTZs2xaxZs1SW4/BmREQEACEhIQgMDFRaRiQS4fPPP2ewtMAXX3wBoYr5Lo4fP47r168zWERERES1wMrKEquWL4aXZ/NKbS8SifDG66PLfRrteW/NmAJ3N9cK7d/AwABvz5yqMKecKhPHj4WPt1eVYtPNzxfjXhulMfHTZQnlPGlHRFTfaWTS7vfff8ezZ8+UlnF1dVUrEUSaYdGiRWjYsKHSMiEhIbh06RKDRURUz61bt05lmcmTJ6Nly5YMlhZo3bo1AgICVJb74osvGCwiIiIiJQwNDattX2Zmplj2yceYMTVAYe628ggEArRq6Y0VSxZg2JBXFeZuK0uDBg2wcvkiDB/6KoxUHL9AIECvnt2wZcMa9OjmB1NTkwp9JlNTEyxdNB+TJrwOfX39Cm3b1N0Nny5dgPfemVnm/Hd1FT9dJpGI2amJiMr6e1FSUlKiaQfl6+uLq1evKi2zdetWzJjByUS1yaJFi7By5UqlZcaMGYM9e/YwWERE9dSzZ8/g7u4OqVRabhl9fX08fPgQLi4uDJiWiIqKgoeHh9J61dPTw9OnT+Hg4MCAERHVU2+//Ta++eYbleVatGiB8PBwBoxgY2OD1NRUleWWLFmCpUuXMmDlKBKLcedOGEKv30RcXDwyMjORX1CAhmZmsLK0hJWVBZwcHdDNzxdWVpaVfh+xWIxbt+/i0eMoZGZlIzsnG0aGhmhib48mTRqjqbtbucNOVlReXh4iIh/gXngkwiPuIzUtDYUFhRBLJDBv1Ah2tjaws7OFnZ0N3Fyc8VLH9pVOotVW/IiIqH7Q07QDCgkJUZmwc3Z2xuTJk1l7Wmbu3LnYtGkTcnNzyy1z4MABxMXF8YIdEVE99dVXXylN7ABAQEAAE3Zaxs3NDRMnTsSOHTvKLSOVSvH111+rvMGHiIiIiKqXoYEBXurYDi91bFej72NgYIBOL3VAp5c61PhnMjExQccO7dCxQzudiR8REdUPGjc85vfff6+yzJw5cyr8mDvVPUtLS0yfPl1pGalUih9++IHBIiKqh6RSKXbu3Kn8xEUoxEcffcRgaaGPPvpI5dx2O3bsUJm0JSIiIiIiIiLSVRqVtCsoKFA5NKKZmRmmTJnCmtNS//vf/1ResNu1axcDRURUDx07dgzJyconIx88eDCaNm3KYGmh5s2bY+DAgUrLJCYmIjAwkMEiIiIiIiIionpJo5J2f/zxB7KyspSWmTRpEho2bMia01Lu7u4YNGiQ0jIPHz7EX3/9xWAREdUz6ty08e677zJQWkyd+lP1tCURERERERERka7SqKSdqqfsAGDatGmsNS2naohMddsCERHpjqysLBw7dkxpGU9PT/Tq1YvB0mL+/v5o3ry50jJHjx5FdnY2g0VERERERERE9Y7GJO3y8vLw559/Ki3Tvn17tG7dmrWm5fr374/GjRsrLXPw4EGUlJQwWEREdSRHnFGr73fkyBFIJBKlZSZNmsSK0QEBAQFK14vFYpUJXCIiIiIiIiIiXaSnKQdy/PhxFBYWKi2j6iIPaUmj09PDuHHjsH79+nLLxMbGIiQkBJ07d2bAiIjqwBfXpqONTU+84vwazA1tavz99u/fr3S9UCjEhAkTWDE6ICAgAJ988glkMpnS9vD6668zWERE9Ux8fLxa5dLT0/Hll18yYISCggK1ysXFxTFYREREpBU0Jmmn6o5qgUCAkSNHssZ0xOjRo5Um7f5tE0zaERHVDYlMjAtxB3Ep/ij8mgyq0eRdUVGRyqftu3fvDgcHB1aMDnB0dISfnx8uXrxYbpnAwECIxWIYGBgwYERE9cijR4/UKpeUlIR33nmHASO1PXz4kEEgIiIiraAxw2OeOnVK6frOnTvzYp0OUac+VbUJIiKqecUlElyIO4hPr7yBAw+3ILMopdrf4+LFiyrvkh4xYgQrQ4eoqs+8vDxcunSJgSIiIiIiIiKiekUjknbh4eEqh8EYOnQoa0uHCAQCDBkyRGmZ0NBQZGVlMVhERBqgJpN36tykMXz4cFaCDlEnCcubd4iIiIiIiIiovtGIpN3Zs2dVlunbty9rS8eoqtPi4mKcP3+egSIi0iA1kbwLCgpSut7b2xtOTk4Mvg5xcXGBl5eX0jKnT59moIiIiIiIiIioXtGIpN2VK1eUrre2tka7du1YWzqmV69eEIlEVWobRERUN15M3mUVpVZqPwUFBbh165bSMn369GHAdZCqer158yYKCwsZKCIiIiIiIiKqN7QiadejRw8IBALWlo5p1KiRymQsk3ZERJrt3+TdiivjK5W8u3btGqRSqdIyvXv3ZqB1kKp6lUgkuHHjBgNFRERERERERPVGnSft0tPT8ejRI6VlfH19WVM6SlXdhoaGoqSkhIEiItJwlU3eXb16VWWZLl26MMA6SJ16Vad9EBERERERERHpijpP2oWFhaksw6Sd7lJVtzk5OYiJiWGgiIi0REWTd7dv31a6vlmzZrC2tmZgdZCtrS3c3Nyq1D6IiIiIiIiIiHRJnSft7t27p3S9QCDgfHY6rH379irLqJPYJSIizaJu8k7Vd/xLL73EYOowVfXLcwAiIiIiIiIiqk/qPGkXHh6udL2LiwtMTU1ZUzqqefPmMDAwUFqGF+yIiLSXsuSdTCZDZGSk0u1btmzJIOowVfUbHh7OYbKJiIiIiIiIqN7Qq+sDePz4sdL1Pj4+rCVdboB6evDw8FD6xOWTJ08YKCIiLfdv8u5S/FH4NRmE3s6vIzMxDwUFBTwPqMdU1W9+fj7i4uLg6OjIYBER1QO2trZq3bTZsGFDvPrqqwwYYf/+/RCLxWq1LSIiIiJtUOdJO1XzlTVr1oy1pOOaN2+uNGkXHR3NIBER6Yjnk3fmcS1Ulvf09GTQdJiXl5fKMtHR0UzaERHVo78LZ8+eVVnOwcEBv/76KwNGsLGxQWpqqspy3t7eDBYRERFphTofHlNVQsbFxYW1pOOcnZ2r1EaIiEj7FJdIcDHspMpyPA/QberUr6obvIiIiIiIiIiIdEWdJu1yc3ORm5urtIyqhA5pP1UX7BITExkkIiIdlJtSpHS9jY0NGjRowEDpMBMTE1hZWSktExsby0ARERERERERUb1Qp8NjpqenqyxjZ2fHWtJxqsaWz8jIwMP0WxAKhQwWEVEtkZUU1/h7FGRJlK5v0qQJK6IeaNKkCdLS0spdr2wdERERERFVn0ePoxCfkFD6upufb6Wvx0VEPkDKP8PXCgQCdO/aRSOOvy6Oi6oP649qwvmLl5CVlQ0AsLGxhm+njnV6PBqftLO0tGSr0XGq7rAvKSnB+ovvwKihPoNFRKRDinIkVfr7QLpB1bmeOueLRERERET/ys3Nw9XQa3j85Ckys7KQk52LRuYN4dDEHg5N7OHs5AhXF47sVZY9+w7i5q07pa+7+HaqdNJu54+/4knUUwC1l1xR5/jr4rjYj6oP649qwtHjJ0vblbGxcf1O2mVmZqosw6Sd7lOnjotypUzaERHpmMJsKc8BiEk7IiIiIqoWRUVF+PGXPTh9JhjFxcpHDvHz7YTpUybCzMyUgauggoICHDx0DNJ/YtzU3RVdu3RmYNiP2K6IqoleXX8JqGJsbMxa0nHqzFcklcgYKCIiHSMVK/9uNzMzY5DqAVX1XFBQwCARERERkVI5OblYtHQl4uIT1Cp/6UoIwiPu473ZM9HSuwUDWAFZ2Tk4cOho6eue3f2YXGE/qnI/Yrsi+k+dJu3EYrHKMgYGBqwlHadOHcskJQwUEZGOKZbKeA5AKutZnfNFIiIiIqq/ZDIZNn21tcxEg76+PpwcHZCbm4uU1DSUlPx3fSkzKwsbt3yLLetXq3VDOVWMfWM7FBf/PbpKZYfYrE/HxX7E+iP6V50m7SQSicoy+vocElHXqZW0K+aTdkREOvejQMUNGUza8TwAYNKOiIiIiJS7duMWbt2+K7fMs3kzTJo4Dm6uztDT+/vyZ3p6Br7/4RdcDblWWi4zMwt79h3EpAnjGMhqNmf2Wzwu9iPWH1El1Gk6WiQSqSyjauxc0n5SqVRlGYFQwEARERHVQwIBzwGIiIiIqHznzl+Ue+3m6owFH81F82bupYkGALC0tMCHc/6Hl3t0lSt/PPA08vM5JDuxH7EfEWmGOn3STp076MViMe+013Hq3EEv0uPjzkREukaoL6jy3wfS/fMAngcSERERAUVFRaU3PYtEejAyMlS5TV5+PvDPMHaGhoZyF97LOid7EvUUCYnJSExKgkQsgYNDEzg7OcLRsQkaGBlV6riTkpLxJCoacfEJMDExho2NNby9PGFsXD3D6BUUFuLGjdv//cYQCPDRh3NgYmxc7jaTJoxDSOgN5P8zd7JMJkN0zDO08PKo8PtLpFKIi4oAAAKBsPRzlZSU4P7DR4iLi0d6RibMGzWCfWM7tPDyKPMhhry8PNx/+BhRT6NhZWmJ5s3c0cS+sdIb2Gq6TZT3WfPy8uWWS6XFyMvL+zv+IlGl28rzdXr3XjiSklKQnZMNhyZN0LypG+ztG9fpcIhVactSqRRF/7QToVBYOoykRCLB/QePEPU0BmKJGD27+8HayqpW+qam9KPqbld12Sdr47uwNtpDbX2nS6RSPHr0BMkpqUhOSYFEIoWlhTksLS1g39gOTo4OFd5nsUyGuLgEPH4ShSdRT1FcXAw7WxvY2dnCo1lTWFpaVHh/YWERSEhMQlp6BhqamaJZM3e4ubrAsAavVWh80q6wsBCmpqY8O9PxE09VVF3YJSIi7aPqhgx1/j6Q9mPSjoiIiEi1eQuWls41ZW7eCNu/2aS0fMi1G1i7bnPp63f/9ya6d+2iUE4mkyH4wl/YvfcA0tMzyt1fp5faY9abU2FiYqLyWPPy8/H9zp9x7cYt5OfnK57f6euj00vtMXb0CNg3tqtSXFJS0iB9bpSuFi08YaXioqypqQlcXZ0RHnG/dNnT6JhKJe2WLP8MDx4+BgBYWJjju6834tade/jx598Q8yxWoXzjxnaYGjAe7dq2BgBcDbmO337fj2ex8QpljY2NMWLoqxg25NVabRPl+WztBty5G6aw/K/LV/HX5asAAC/P5vh06UIAwIJPVuBJVDSAv5/O+nrT56XbFBQUYPKMd0pfjxk5DMOGDMTvBw7heOAphQQOAJiamGD61Ino2qVzldqMsuOqqba8btPXCL12AwBgZ2uDrzZ9jgcPH+Orb7fLzSHXzN2tNGlXE31TE/tRdbaruu6TNfldWJvtoaa/03NychF4Kgh/njyDzKyscsu5u7miT++X8UqvHhCpkbAPvnAJ23f+iIKCwjLX6+npwf+VnhgxdJBaybuzwRewd98hpKSmKqwTCoVo4emBD+b+D2Y1kLuq06SdOsm4zMxMWFtb8+xMh2VmZqosY2AsYqCIiHSMnqFQxYlcDoNUD2RnZytdXxuTmRMRERFpOrFYUua/y/PvEyrKFMtk+GztBoV5rMoSEnoDT5/GYO67s9CsqVu55R4/eYp1m75CcnJK+Z9FIsHFS1cREnoDARNeR78+r1Q6Lunp6XKv3d1c1drO0sJc7nVWduV+e0il8tP6XLkaig1bvi13up/ExCR8vn4LVq9cgktXQrHvwKFy952fn4+fd/8OkUgPg1/tVyttojq3l0qLS58ETEmRv+hdUiI/XU5ySgo2f7UVFy9dLXd/uXl52LD5G9y9F4HJAeMq/ZSLsuOqqbb84tRAd++FY+Wa9eVOGVQTfVNT+1F1tqu67pM11X5quz3U5Hd6QmISlq/8vMxE2IueRD3F1u27EBH5AO+8Pb3cpxzFYjG+3/Uzgs6eV9F2pAg8GYQrIdfwxWfLYW7eqNyyv/62DwcOHS13vUwmQ1hEJJav/BxLFs6DqWn1JUmBOp7TzuqFx33V+dIg3aNOHRuZ6TNQREQ6xqihPs8BSGU9W1paMkhERERENeDX3/YpXAS2tDBHCy8PNHV3hekLT2okp6Ri2co1yC7nwnxI6HUs/GSFwsVdoUAAMzPFG/fFEgm+2/GjygutSs8lMzJfOH71hj6LjUuQe+3k2KTK8czIyMT6zd+Umxx4/nPP+3iJ0uTA8374eTdu3bmn023x9JlguYSdnkgEW1ubMi+Enz5zDjt/+LVGj6cm23JqWjrWrNtcbsKuJvqmNvWj6lRXfbK6209tt4ea7AdfbPhSIWFnaGgAN1cXeLfwLPNJvfMXL2HfwcPl7nPV2g1lvmeDBkawtbGG8IVkX2ZmFjZ++S1kMlmZ+8vPz1dI2OmJRDDQV7yGFfU0GmvWbar2tlunT9qpcxEmVY2sK2k3VXXcwLgB5nfZzkAREdWiLTffQ1Fxfo2+h5GZ8tOQtLQ0VkQ9wKQdERERUe0rKirC8cBTpa/19fUx++0Z8O3cUe5phlt37mHb9z+UXrQtKCjEH0eOY+L4sQr7277rZ7kh9qytLDFjagB8vL1gaGiI9PQMPHj4CLt+2o3UtP/OAbfv/Am+nTsqnT+rPN27dUHnl9qXvjYwVD2v2/0HjxAd80xumauLc7XE9d+LwN38OmP40EFwdHRARkYGDh46hj9PnSkt93ycnByb4I3Xx8DDoxlycnIReu0Gjp04KZdIuXDxEtq2blmnbWbpovmQyWRISk7BvAVLS5d3eqk93p4xFQAgElXt+RAjQ0OMf300/F/pCf1/LpA/ehyFn37Zg7CIyNJyZ84GY2B/fzg7OdZI36jJtlxcXFyaRNITieDj7QUXF2eYN2qI5s2aVnvf1PR+VNPtqrb7ZHW3n7poDzXVD27dvivXZvT19TFx/Fj079tb7rPEJyTih5924/rN/+ZZvHb9FkaPGKpwnCHXbuBeWITcsh7dusC/98vwbN4MIpEIRUVF2PP7QRw5/idK/pnT815YBK6EXIOfbyelcWjbphVeHzMSLs6OEAqFiE9IxM4ffsHt54Z0jYh8gMSkZDS2s62276E6TdqZm5tDT09P6Z0Fz54945mUjlNVx43tGsPBtBkDRURUi4SCmn8Yv0Ej5U/axcXFsSLqAVX1rM7IDERERERUMXfvhUMi+W9IxUED+6KL70sK5dq2bomPPngXc+ctKl324gVSADh05ITcPEsezZti4fy5cvMqWVpawLfzS3Bzc8WSFauRmvr3TXoSiQQX/7pSqWEy9fX0oK+n/uXNvLw8bPl6W+mFW+DvC9FVnVvveZMnjsOrA/o+t38rTJ8yEQmJSQpzd7X0aYFFH70PvX8+g5mpKYYOHojmzZrik+WflZaLiHxQ523m32HrjY3lhzNsYGRULUPDiUQiLJg/F94tPOWWN2vqhkULPpCb+0xWUoIfftqNxQs+rPbPWVtt2c7WBnPfnYWm7q5yy69dv1mtfVPT+1FNt6va7pPV3X6q+7u6LvvB/YeP5F6/PmYEBvTzV3jvJvaN8cGc/+HdDxaUJiGfRsdALBbLzXlfXFyMn3f/Lrdt61Y++N9b0yF8bg48Q0NDTHzjNWRmZeH8xcty8VGWtHtt9AiMGjFEbpmjQxN8NG8OPlm2Cg8fPSldfvvOPTSuwlDPL6rT4TGFQiEcHZXfEREdHc0zKR2nqo6dnZ0ZJCIiHWRqY6R0fVpaWpXnXSDNlpOTg4yMDKVlVJ0rEhEREVHFJSQmyb0WKLlpz9nJUW5upMSkZLn1GRmZ+OPI8dLXeiIRZr05Ve7i7vPsbG0wb+5suWVngy/U+GeOjYvHR4uWKxz/229OlbvAWxVurs5lXoQGgA7t2si91hOJMH3KxNLkwPO8W3jC6LmnnbJUzAOtC155ubtCwu5f+np6mD5lIkQiUemy23fDlM5JVxm11ZYN9PWxdPFHCgm76u6b2tqPqlNt9smaaD910R5qqh8kJsofT9cuncs9Bn19fXg0cy99XVxcjLw8+dGgzp3/C/Hx/w3RamVpiTnvvFVuOxw0UH4ewnvhkUpjOXzoq+V+H/Xo5qcQs+qkV9cdx8XFBU+fPi13vbJ1pBuioqKUrmfSjohIu+lLANN8ERrmC2GWL4RZngjmBQbolGqJQIQr3TYmJgYtWrRgEHWUOjdnubi4MFBERERE1czcvJHc68A/T6Ndm1Zo4eVRZvlVyxfLPVXzvLth4RCLxaWvX+7ZDQ4Oyue2cndzQVN3Nzx+8vc1oUePo5CUlAy7ahxe7F9SqRR/HD6OfQcPK4z21de/F1q38qm29xo+dFC5F4xfjLm7uyscmtiXuy9rayvExsUDAIqKxJBIpRV6GkrbDOzfR+l6+8Z28PRohvCI+6XLEpKSYGNjXW3HUFttefCg/rCxtqrxvqmt/ag61WafrIn2Uxftoab6wYhhg9C2zd9DihoZGsHKqvypMIplMqSkKp9KI/K+/NOOvXv1KHOuvf+O0RWtfLwR8c92ubm55X6vjhg2SO4mgbK+j56XnZNTre22zr/pXV1dERwcXO768PBwnknpuIiICJVthIiINFAJYFrwbyJOCLN80T///+ff/yw3kpR9ghyXp3rOvMjISCbtdFhkZKTKMkzaEREREVU/nxZecq/zCwqweNkqtPD0QM8eXdGuTSu5C6rKnqCJT5B/EqR5s6ZqHYOnR7PSC7wAEJeQWO1Ju5u37mDnT7vlnsb4V78+r2DSxHHV+n5N7Mu/4G/0wjxhquY/en6OJ+C/ubl0UcOGZnBydFBZzsnRQS5pl5iYjNYtqy9ZVFttuV2bVrXSN7W1H2lrn6yJ9lMX7aGm+oGTo4PSfl5QUIDklFQkJCZh/8HDiHoao/T9XnySsEP7tiqPccmieWq2m8ZK1784dGtRUVG1tts6T9p5eXkpXR8ZGYni4mKlmU3SXvHx8SqHxeLFWiKi2qcvBoyzRaVPxpnlP5eM+ycxZ1oghLBEUPmTZ+MGaCASoeC5iY1fdO/ePQwfPpwVoqPCwsJUnAibwl7JjywiIiIiqhxLSwu88nIPnDl3Xm55xP0HpU8h2NraoHVLH7Rv2wqtWnqXzj31ooSERLnXP/2yB8cDT6k8hvgXtqvO4cXiExKx66fduHHztsK6hmZmmDr5DaVDs1WWjbWl2mWtra3ZEP/R1M1VrXIvXvCv7uEga6stOzo41Erf1NZ+pK19sibaT120h9roBzHP4hAWHoGIyPtITEpBckoKcnMrNj1K0nP9XyAQwM21+kbrU/UEr1BYs7mqOk/atWzZUun6oqIi3L9/H97e3vwLpoPu3LmjsoyPjw8DRURUg6Qx0Sg4cRSy9HTIMtIhy8zA7EJjAMY1+r4CgQAejRridnr5N2/cu3ePFaTDVNWvt7e3wt2MRERERKSaTKZ6eLSZ0ydBJBLiVNC5MtcnJ6fg9JlzOH3mHEQiEfx8O2Ho4IFwdXGSKxefIP/0TW5eHnIrMTe1RCKphs8tw+GjJ/Db3gOQvnBzoEAgQJ/eL2Pc2FEKT0lUB0NDg3LnfCqLvp5I49pEXVE2pN3zHF8Yoi89PaNaj6O22rKhkWGt9E1t7Efa3Cdrqv3UdnuoyX4QFxePbd//iLCIyCrFWiwWI/25pKCpqUm1PXVqaGgAM1PTOm27dZ60Uychc/XqVSbtdNTVq1eVrheJRPD09GSgiIhqkMjBEcUx0Sh+FlPr7+1l3khp0i40NJQVpMNCQkKqfJ5IRERERIoKCgpUlhEKhXhz2iT069Mbx078iWs3biEnJ7fMssXFxbjw12VcvhKCRQs+QEvv/0ZFys7OrZZjrurwj2lp6dj89TaEhSteDG7p0wITx78Gd7eaG3pdJNLT+jZRVwwMDNQr+ML9fPr61RtzTWnL1dU3tbEfaXOfrKn2U9vtoaY+x9174Vi5ep1CIvh5eiIRnJ2d4N3CE+ER9/Ek6mmZ5TKzsuReV+fTpprwXa4Rc9pZWloiPb38iQWvXLmCyZMn84xLB125ckXp+pYtW8LQ0JCBIiKqQQKRCKZT30TWskVASe3efdnSwhx7lKyPiopCcnIybG1tWVE6JiEhATExyhPFbdq0YaCIiIiIKiFXjfmj/+Xq4oRZM6dBJpPh0eMohIVH4F54JCIiH0AsFsuVlRYXY+0Xm7Fp3WewsDAHANjaWCPtuet60yZPgLOzY4WPuZm7W6U/b0ZGJhZ88qnccQCAtbUVpk2egI5qzHXENlF3kpJS1CqXliZ/w2d1DwupCW25Ovsm+1Htqun2U1vtoSY+R05OLjZ9uVUuYScQCNC2TSu0bukNF2cn2NnZwtrKsnSatM1fbSs3aWfeqBEEAgFK/rmGlZ2VrVNtqc6TdgKBAJ06dUJgYGC5ZS5evMher4OKi9zHRlEAACAASURBVItx+fJlpWV8fX0ZKCKi2jghcG8Ko74DUPjn8Vp93442VirLXLp0CcOGDWMl6RhV5wA8DyAiIiKqvKRKzPUlFArh0bwpPJo3xfChgyCRSBAWHokz587j0pX/RsDILyhAeOQDdO3SCQBgb9+4dG4lADA3bwRvr9obNalILMbqLzYpJBr8X3kZAW+MrbH5vupDm6gt0c+eqVUuLU2+jl8cLrOq6rotV3ffZD+qXbXVfmq6PdTE57h245bc03GGhgb4eN4cpU8C5uTklLvOwMAA1lZWSElNBQAUFhUhLy9P5XCokfcfID4hqfR1V7/OMFT3Sd9apBHPbfv6+ipN2oWHhyMuLg4OSibpJO0TEhKCrBceZS2rbRARUe0wHjUG4mshkKWl1tp7tra0gL5QCImS4UNOnz7NpJ0OOn36tNL1hoaGaNeuHQNFREREBOD5aX4LCgpQWFgEIyXzYt0Lj1C6vx9+/g1FRUUAABdnJ/Tr84pCGX19fbRt0wpt27SCvf1+7D94pHTd4ydRpReCHZo0ltsu6mk0fDt1VH0+eCa49CkKfX19jBs7slKjLe3ZewCPn0SVvhaJRPhwzv/QsUM7tokKtIm6lJ2dg6zsbDRq2FBpues3bsm9buXTolqPo67bcnX3Tfaj2lUT7acu2sP/2bvv6Ciqtw/g351tSTY9hAAJSQiBhGqAhCBNEEIVpCqIlBCKYEORH4IoiihiQbDRe1MERURCL1YgICFAQFoKICUE0tvuzLx/+IIG0meT7CbfzzmcQ3bvPDv7zN2d2Xnm3imP93H23F/52g7q37fIgp0oijj714UiX69ObY/7RTsAOH3mHEJbtypymUVLV+Hqtb8BADY2NujY/lGL7EuCJaxEhw4dFJ/YIeuzZ8+eYtu0b9+eiSIiqqgffXobGEZFVOhr6tVqNHN1Vry/oKp3HNCqVauS39uCiKgQGTeNTEIhTNeuMglEVuS/owdkWUZ8QkKhbQ/+/Bvu3k0pMt7hI8ewe+8B7N57ABu+2QyTyVRk+9bB+U+E/ncqtjq1a+c/ztt78P5J5sIkJd3GspVr76/DoV9+uz8lWmmIoogDP+efoWvi+IhqUWgwd5+obIePHCvy+dizf+GvCxfv/+3q6oLatWuZdR0qsy+Xx2eTn6OKVR79pzL6Q3m8j7sp+QfuFPfZ/evCReTk5BTZxse7br6/d+7ZV2T7y3Hx9wt2ANCsaWNoNJZ5L1KLKNq1a9eu2CG227dv5ye/ivnpp5+KfN7X1xf+/v5MFBFRBdI90gK6R9tV6Gs+VsujyOfPnz+PuLg4bpwq5NKlS7h48WKRbbp27cpEEZFif8y/hWNLbiP1ah6T8YD0BZ8g7cP3YbzwF5NBZAWcnZ3y/X346PEC28UnXMGS5auLjVe/vu/9/2dmZuH0maJHYT14XyG/ev8u3yKoGerU+fckb1p6On74MbLQWJIkYc36b/KdfG7bpnWZTp6ejDmN9PSM+3+3DmmJxzq0ZZ8oQ58oq7T/5F+JlWs24OKlywU+l5mVhbUb8t8N3dyj7Cq7L5fHZ9OaP0fm6lcVqTz6T2X0h/J4H+418t8aJSkpudB4cfEJmP/ZwmLXs3ev7tBptff/PnU6FkeP/VlgW5PJhIVLVuZ/n480s9i+ZBGlRL1ej44dO2LXrl2FtomMjER2djbnz60irly5gqioqCLbhIWFMVFERJXAMGwkUv/8Fba5qgp5vU51auHT00UfeH733XeYPHkyN04V8d133xXbhscBRGQuN05m48bJbNR6xBYNejnCyYujeO8xno6B8XQMtE2bw7b/QGgbBDAp9JC1a9fi6tX8IzOnTJlisVenV1WBDf1xIjrm/t/bd+yCg70BYV06w9HRAX9fv4GYU2ewbXtkiUZWNGkUmG9k02dfLsGEcaMREpx/ZE1uXh6OHY/GyjUb8j3ewN/v/v81Gg3GjHoWs97/6P5jm7ZsRWpaGkaNeAba//SVpNvJ+OzLxTh77t/7Jem0WvTsXrYLth48QZuenoFVazeWOk7bNiFo2MC/WveJkrK1scn394noGJyOPYvAhg0gCAIEoWxjREwmEz6a9wXGjxmJRo0C7r/OlavX8PGnX+Da39fz9Zk+vXuaPaeV2ZfL47NpTZ+j8upXFak8+k9l9IfyeB++Pt75/v762y3w8fFC86ZN8r2H334/ghWr1iGngJF9qWnpcHH5d6YmN1cX9OrZDVu3/Tsw6ONP//kO6dDu0fsz99xKSsKXi5YjLv7f0ci2tjYIseCRpBZzhNW9e/cii3aZmZnYtWsX72lTRXz//feQZbnINj169GCiiIgqgeDoiIOP5qHnQX2FvF5IDTcYNBpkFjHNA4t2VcuWLVuKfN7BwQGhoaFMFBGZFYt3hWPxjorywgsvIC0tLd9jr7zyCot2Fax9u0fx7ZYfYBLF+49t3PQdNm76Dra2NsjOzilVvLAunbB7735cufrPVGFp6emY+8kC+HjXhWed2nByckRcfAIuXryc7zXvLVvXyzPfY82bNUGb0OB8J5d37dmPfQd+hnddL6hUKmRlZeHGzVsPnQ8KHznsoXglde3a9Xx/nz13Pt/J45LyrFPb6op25u4TJeXk5Ai9Xofc3H8LgW+/O/ef33bBLTF18ktljp185w7e//BTqNVq+PrURUpKGpLv3HmoXUT4cPj61C2X91dZfbm8PpvW8jkqz35VkczdfyqjP5TH+wgJboENX29GRmYmACA3Nw+z3vsI/vXroZZHTaSlZ+DCxUtFfm/NnvMxvL29MD5iJDw8agIA+j/ZGwcP/YqU1H+m35QkCQuXrMSS5WtQ18sTObm5uFnAOj7/3Jh8BUBLYzEl6v79+xfbZt26dTxKqyKK25a2trbo3r07E0VEVEkuNdIjxc+1Ql5Lp1aji2ftItv88ccfSCjiHg1kPeLi4nD06NEi2/Tq1Qva/0xzQURkTjdOZuOXOTc5bWYBjKdjkPbuTE6bSWUmSRLS0tLy/cvIyGBizMCjpjumvvZyvqnA7nnwJKedrS1eeWlCkfE0Gg3Gjw1/qPiakHgFvx8+ishde3HurwsPnQSu61UHo4YPLTDmxHER6Ng+/5R6JpMJl+PicelyHK7fuPnQidM+vXsgrEunsn+n37rFPmGmPlEaHdo9WuDj6enpZYrnXdcLbq7//v4URRGXLscXWLDr/Fh7dOncsVxzWxl9uTw/m9byOTJ3v6os5uw/ldEfyuN9uDg7Y9yYkQ89fvFSHH79/QhiTp3J973VKKAhPnz/nXxtU1JTEXPqDK7fuHn/MYOdHT547y3416+Xr60oiohPSMSNAtax/5O90aZ1sEX3IYsp2vn6+qJly5ZFtvnxxx+RnJzMIzUrd+bMmWKnxuzRo0e+G+oSEVHFmhqyHL4TZwK6ihmF0Mfbq8jnZVnGmjVruGGqgNWrVxc72n7gwIFMFBGVOxbvCsfiHZXVqVOn4OTklO9fSEgIE2MmLYKaY/rrr8K1iNEB7du2wSdz34Wfr0+x8QIbNsDc92bCx7v4EUt6vQ5PD+qHD957G3p9wTNy2NnZ4qXnx+GVlybAzs6uyHh1atfC1MkvY+SzQ8qcD0mSkJGRWanbRKNRl3lZmwemA7SEPvHfwoBOqy10SsKhTw1EQMMGZstN7VoeePftaajn611oGxcXZzz/XAQmjo9QtP4lWS9z9+WS5rW8PpuW/jkyZ7+q7M9kefSfiuwP5fk+2rZpjamTX85XoH+Qq6sLxoQPx6yZ0+BXzwe9ehR/24wabm54d+Z0hHXpBI268O0fGNAA770zA8OGDDZrvykPKrm4syYVaM6cOZg+fXqRbRYsWICXXrKO4bBUsNdeew2ffPJJkW3WrVuHYcOGMVlERJUse8ePyPp6fbm/TrrRiIBNW5EnSYW28fPzw8WLF6FSqbhhrJQsy/Dz80N8fHyRP5Ru377Ni3eIyCz2TLuG3DSpRG2r27SZd//3CqQb10vUtqKmzZw4cSIWLlxYbLtGjRohNjaWHbyCODk5PTQ9ZnZ2doEnN0+ePImgoKB8jwUGBuLs2bPlsm7u7u64fft2se1mzpyJt99+u2p9hu+m4FJcPOLjE6FWC/CsUxs+3nXvTxlWGpIkIT4hEWdiz+HS5ThkZGbBZDKhhpsrPGq6w8OjJpo2bgRXV5cSx8zJyUV8QgIux/3zLyc3B64uLnBzc0XjwIAy3WeJKq5PlNSNm7fw99/XIcsyDPYG1PXyhKGYk/sAkJWVjRER/478Cw1phSmvvggAOBN7DrHn/sLt28lwsLeHwWAHL09PNGvaGDY2+grPa2X25fL4bFqDsvYrS2TO/lOZ/cGc7yM7JwfR0adw5eo13Lh5E3q9HjXcXNGgQX00bdzooaL25bh4xMUnQJJk1HSvgUaBDe/fr+5BRpMJCQlX7o8EdHFxRu1aHqhTuxa8POtYTb+xqKLdlStX4OvrC6mIE3YNGzbEuXPneMLOSmVlZcHLywt3794ttI2DgwNu3LhRbAWfiIjKnyxJSJ05HWJCfLm/1uiff8e2hCtFttm5cyenT7ZikZGR6NWrV5FthgwZgo0bNzJZRGQWpSna3VNdinelKdrdU97FOxbtLBOLdkRkbkUV7YiIqjvBklambt266NKlS5Ftzp8/j507d3LLWak1a9YUWbADgKeeeooFOyIiC6ESBNhHjAcq4GKZZx6Yg7wgCxYs4EaxYvPnzy+2zahRo5goIqpUnDazcJw2k4iIiIiofAmWtkLh4eHFtiluakWyTJIklehkXUn6ABERVdB3d3YWcn//FaiAgfmda3uglm3Rc8jv3LkT586d44axQrGxsdizZ0+RbTw9PREWFsZkEZFFYPGucCzeERERERGVD4sr2g0cOBAeHh5Fttm3bx9+//13bj0rs2nTJvz1V9E/6B555BG0a9eOySIisgB5p04iZepk5Oz8qUJeTy0IGN6yRZFtZFnG+++/z41jhd5//30UNyv72LFji70pOxFRRWPxrnAs3hERERERmZfFnRXR6XSYMGFCse1mzZrFrWdFZFnG7Nmzi2338ssvM1lERJVMSrmL9M/mIf2jOZBT7lboa78w54NCbyh8z4YNG3Dx4kVuKCty/vx5fP3118UeAz733HNMFhFZLBbvCledineyLOPKlSvYv38/tmzZgiNHjiApKalEy6anpyMmJgY7duzAli1b8Msvv+DixYuQJImdiIiIiIgAABpLXKnnnnsOc+bMQW5ubqFtdu3ahUOHDuGxxx7jVrQC69atw5kzZ4pso3eoAZugp2EUZWjVKiaNiKgS5P72CzLXrICcnV3hr61r2x5uHTriqaeewrp16wptJ4oiZs2ahTVr1nCDWYl33nkHoigW2WbIkCHFzrZARGQJbpzMxo2T2aj1iC0a9HKEk5eOSfl/xtMxMJ6OgbZpc9j2HwhtgwCrWfeMjAw88cQT+R7r1q0bpk+fDgBITk7G/Pnz8cUXXyAlJeWh5b28vDBjxgyMHj0aWq0233M7duzA4sWLERkZCaPRWOCyQ4cOxdSpU+Hm5lbkeiYkJGDkyJH5HgsICMDixYtL/F5XrVqFVatW3f9bEAR88sknaNGiRZlyN2XKFERFRd3P44MSExPRqVOnfI/t27cParWaHxoiIiKiB1hk0c7DwwPh4eFYtGhRke0mT56MqKgoqFQs8Fiy7OxsvPHGG8W2q9/rJXz7lwa//23EhBANGtfk9FhERBVFvP43MlYsgemvyrlfnMrGFoYhz97fv69fv77IqRTXrVuHSZMmoWXLltx4Fu7YsWPYuHFj0dtfpcJrr73GZBGRVWHxrnDWWLwzmUw4dOhQvseuXr2K6dOnY/PmzQgPDy+wIPXfts899xw+++wz7N+/Hx4eHkhNTcXzzz+P9evXF/naV69exUcffYSNGzdi8+bNCA0NLbRtVlbWQ+t59uzZUhXt4uLiHoqxc+fOMhftTp48+VC84tZZroD7JROR5RIEAXW9PO//3cDfj0khIvp/GktdsWnTpmH58uUFXoV2z/Hjx7Fu3ToMHz6cW9KCzZs3D1euXCmyjdbgAr/uEwEA19KBGftN6OEvYFhzNQw6FmWJiMqLbDIhZ+8uZG3aCJhMlbYetgMGQXB2BgAEBQWhb9+++OGHHwpfb1nG5MmTceDAAW5ECzd58uRiT8z169cPzZo1Y7KIyCqxeFc4ax55d8/ixYsxYcKEEheZYmNj0adPH+zduxddu3bFsWPHSvxaV69eRZ8+fXDq1CmOPieiKs3GRo9PP3qPiSAiKoDFDmXy9vbGqFGjim03ZcqUAqemIMsQFxeH994rfifs3/tlaG0d8j2286KEF34y4vAVzu9PRFQejJcuIPXN15G1YW2lFuzUnl6wCeuR77G33nqr2OUOHjyIb775hhvSgm3YsAE///xzse1Ksr2JiCwd73lXxDGHld7z7tKlS5g4cWKpR4VFRUXB2dm5VAW7e5KSkjBp0iSr2r6c5pKIiIjIfDSWvHIzZ87EunXrkF3EfXVu3ryJqVOnlmoqCKo4EydOLHL7AYDeyQP+PV8s8LnUXODD30xoUVuF8cEa1DRw1B0RkVJyTg6ytn2PnJ+2ARYwNZFhxGioHjjZ07JlS/Tv3x/ff/99kctOmjQJ3bt3h/P/j9Ijy3H37l288sorxbYbOHAggoKCmDAiK2XMlpB0Nsdi1080Vvx+7v7Iu2Z6+HrHw8HFaJnHA5Vw/9r8I+8GQdugocX3cUn69yJSg8GAoUOHIjg4GE5OTjh16hS+/vprXL58+eH8PnCMpdVqMWjQILRo0QJeXl64evUqVq1ahdjY2IeW3bp1KzIyMmBvb28V3wMLFizA7t27AfwzVeayZcvyPe/h4YE333wz32Ms9BEREREVzKKLdp6enpg8eTJmz55dZLulS5di6NChD93YmCrX2rVrsXPnzmLbNRo8Exqbon+MnLgu4+UdRjz7iBo9/AWoBRbviIjKIi/6BDJXL4OUnKw4lsrOADkrU1EM3aPtoG3UuMDn5s6di+3btxc5VfaNGzd48Y6FmjJlCm7dulX09tfpMHfuXCaLyIrlpIj4c3kyE1HQPupULm7E1IKr6TS88vbAIF1nUv6fNRbvOnbsiJUrV8LP79/7Lg0ZMgQvvfQSunbtitOnTxe6bGhoKJYuXfrQVNCTJ0/GpEmT8Pnnn+f/XOXkIDIyEoMHD7aK7dmwYUM0bPjPNiyoaOfi4oLnn3+eHZ+IiIioBARLX8GpU6eiVq1aRbaRZRkjR45Eamoqt6iFSExMxIsvvlhsOwevxvDtPKpEMXNFYPmfIv6324RLdzhlJhFRaUhpaUj/cgHS5801S8FO3/4xOH/yGTR+9csexMYGhiHDCn26QYMGmDBhQrFhlixZgh07dnAjW5Dt27dj+fLlxbZ7/vnnUb9+fSaMiKoulQp3tM0QY/cK/rIZgUyhNnPyH/9Mm/nW/0+bed5i17Nz5844ePBgvoLdPR4eHli6dGmRxzOHDh0q8N6tgiDg448/hr+//0PPnTp1ih2EiIiIqBqy+KKdvb09Pv7442LbJSYm8sotCyFJEkaMGFGiImrQ6M+gEko3LUZcioype0xYE21CjklmwomIipF75A+kTH0FeUf+UBxLXccTjm++A/txEyAYDDCMHgcIZTucsOs3CIKLa5FtZs6cCXd392JjRURE4Pbt29zYFiApKQkRERHFtqtZs+ZDU2UREVVZLN4V6V7xLu/EcYtbN0EQMG/ePKhUhc/20rp1a7i6FnxM8/nnn0Ov1xe6rE6nQ9u2bQvcnxIRERFR9SNYw0oOGzYMYWFhxbZbv359kVe4UcWYOXMmDh06VGw7n04jUaNRhzK9hiQDW89JeGmHEcf+5qg7IqKCiLduIvWD2cj4cgHkTGXTWEKthk2vPnCaPRfaBgH3H9Z4+8C2d9/Sh6vjCZtuPYpt5+rqik8++aTYdjdu3MCIESMeun8MVSxJkjB8+PBip8UEgE8//RQuLi5MGhFVLyzeFb0fuWN5U62OGDGi2HuvCoIAT0/Phx5v3bo1unfvXuxr3Jta8r9YtCMiIiKqngRrWdGvvvoKNjY2xbZ78cUXcfz4cW7ZSvLTTz/hvffeK7adW40aGD5Z+T1sbmcB7/9swvw/TLibzRO1REQAIEsSsvfsRMq012CKPa04nsa/IZxmz4VhyDCoNA/fDtf2yQEQPGqVKqZhRHiBsQoyfPhwdOnSpdh2kZGRePfdd9kBKtGsWbOwa9euYtuFhYXhmWeeYcKIqPp6oHiXrXJnTixUjx49StSuoAtRAgMDS7Ssvf3D93jPyspi8omIiIiqIasp2vn7+2Pu3OKLPLm5uRgwYABu3LjBrVvBzp07h2effbZEoxyWLV2Kd5/wwLQOGrjZKX/tnxMkvLjDiH2XRY6yIKJqzRQfh9SZ05G1dhVgNCoLZmMDuyHD4PjmO9B4ehXaTKXTwX702BKH1YU+Cm3jpqValSVLlhR4QutB77zzDu9vV0m2b9+OWbNmFdvOwcEBixcvZsKIiABo5XQ4iPHQyXeZDAvl7e1dwsOmhy8yLugeeERERERERRGsaWVffPHFEk2TmZiYiD59+vDKtAqUlJSEXr16ISUlpdi2o0ePRr9+/QAAIZ4CPuupRXd/ASqF65BlBL48KuKNfSZcTWXhjoiqFzkvD1nffYvUmdMhJsQrjqdt0hTOH8yDba8+Rd7D5X77Rk2gf6xz8YH1etgNHV7q9fHz88P8+fOLbSdJEoYMGYKYmBh2igoUHR2NoUOHlujCmQULFqBevXpMGhFVa1opDT45P6JF5hzUMf4MNUxMioXy8fEp87Is2hERERFRaVlV0U6lUmHlypWF3uD5v44dO4ZnnnkGoihyK5ezrKws9O3bF3FxcSX60fLgSVdbrQrjgzX4IEwDX2eV4vU5d1vGq7uM+PaMCKPI4h0RVX15p08hZdpryN66BVA42ljl5AT7FybBceoMqEuwv/0vuyHPQuXkVHSbfgNLHfeeiIgI9O1b/P3z0tPT0bt3b/z999/sHBXg2rVreOKJJ5CRkVFs2/79+yM8PJxJI6Jqi8W6Io5BHByh8W9oWdtLq0WtWrXKvLydnR03LBERERGVimBtK+zp6Yn169dDEIpf9R9++AHh4eGcLrEc5ebm4sknn8Thw4eLbWtjY4PNmzfDwcGhwOcbuAn4qJsGzzRTQ6dWtl4mCdh4SsQrO404c0vihiKiKknKyEDGqmVI//A9SEm3FMfTPdoOzh/Mg751m7IdVBgMMAwvvCAj1K4Dm+69FK3jihUrSnTF+9WrVxEWFobbt2+zo5SjpKQkhIWF4dq1a8W29fX1xbJly5g0IqqWWKwrnMrBEXZDhsFl3ufQ+Pha1Lq5uLiU6NyDtcnLy2PHIyIiIrJQVnn02aNHD7z55pslart27VpMnDiRW7ocGI1GPPXUU9i7d2+J2n/55Zdo0aJFkW3UggqDmqixoKcWQbWUj7r7Ox14c78Ji6JMyMhj8ZaIqo6841FImfoKcvfvVRxL8KgFx2lvwWHCixAMBkWx9K3bQBvUssDnDMPDodJoFMV3c3PD5s2bodfri20bGxuL7t27IzU1lR2mHKSkpKB79+44e/Zs8f1Cr8fmzZtLNFsCEVFVwmJd4f5brLPt1QeqEuzbyTzu3uU9FImIiIgslcZaV/ytt97CsWPH8NNPPxXbdtGiRTCZTFi8eHGVvEquMuTk5GDw4MHYvn17idqPHz8eo0ePLnF8D3sV3uqkxYE4EWuiRaTmKlvf3ZckHLkqYWywBm3rsg8QkfUSb99G5qplMMZEKw8mCLAJ6w67wUOh0unMto6GURFIeT0WyMm5/5guJBS6ps3MEj84OBifffYZxo8fX2zbP//8E926dcPOnTvh4uLCDmQmd+7cQY8ePXDixIkStf/iiy/QqlUrJo6oCrL30CBsTh2LXb9D791AXkbFz7yhdxTg18Ue3i2doNYOBzDc4nKT+s4MSLduVvjrqhwcYdu7D2y6dGOhrpKwaEdERERkuay2aCcIAr7++mt06NAB0dHFn7hctmwZMjMzsWbNGmg0Gm55BTIzM9G3b1/s37+/RO27deuGL774okyv1bmeGsF1BKw4IeJQvLIf26m5wMe/mRBU65976HnYq7gxichqyJKE3EP7kbl+LZCXqzieup4f7CPGQ+PtY/Z1Vbu6wfDUUGSuWfnPAzo97J4x78nKcePGITY2FgsWLCi27dGjR9GpUyfs2bMHNWvWZGdS6ObNmwgLC8OpU6dK1P7VV1/FmDFjmDiiKkolqKB3VFvw+lXs6+kdBdTv6gifDgaodRZ+sWAFX9DKYp1l7cuJiIiIyEIP06155e3t7bF9+3Z4enqWqP3GjRvRu3dvpKWlccuX0fXr19GxY8cSF+yaNm2Kb7/9VlGh1EGvwsttNHj3cQ1qOyh/D9E3ZLwcacSPf4kQJU6ZSUSWz3QlEWmz3kTmymXKC3Y6PWwHPQ2nmbPLpWB3j/7xMGj8GwAAbJ/sD7VbDbO/xrx589C3b98StY2JiUH79u1x6dIldigFLly4gPbt25e4YNevXz989NFHTBwRVXl6RwGNBzjj8Xdqw6+Lg+UX7CoQp8Esf6IolrhtXl4eoqKimDQiIiIiC2X1vyQ8PT0RGRlZ4nuk7N69G+3atUNiYiK3finFxMQgNDQUf/75Z4na+/j4IDIyEo6OjmZ5/SY1BczvoUX/RgI0CntungisPCFiym4TLt6RuHGJyCLJRiOyfvgOqW++DtNl5cUmTWAjOH/wMez69oeqnK+uVwkCDBHjofb0gm3PJ8rnIEYQsGHDBoSGhpao/YULF9CmTRv89ttv7Fxl8Ouvv+LRRx/FxYsXS9S+TZs2WL9+PacmJ6IqjcW6Io4FWKwrn7yqHp4xJjk5Gbdv3y7R8ocPH0ZWVhYTSUREmgtLJQAAIABJREFURGShqsQvimbNmmHnzp1wcCjZMKzTp08jODgY+/btYw8ooa+//hpt27bFlStXStS+du3a2LdvH7y8vMy6Hlq1CsMf0WBedy0Cayif3jI+RcbU3SasjjYh28hRd0RkOYxnY5Hyxv+QvWUTICm7uEDl4AD7516A0/SZUNdwr7D3oPH0guOMd6Aqx2mpDQYDIiMj0bx58xK1v337Nrp06YKVK1eyk5XC8uXL0bVrVyQnJ5eofVBQECIjI2FnZ8fkEVGVpHNgsa7w4w4W68qTm5tbgY+X9OLa1atXM4lEREREFqzK/LIICQnB9u3bYTAYStQ+KSkJ3bt3x/vvvw9ZZrGmMEajES+//DKGDh2KzMzMEi3j7u6OvXv3on79+uW2Xl5OKrzXRYMxLdWw0yqLJQP44ZyElyKN+PM6R90RUeWSMjORuXYV0ubMgnTjuuJ4upBQOH8wD/q27SvnQKOE+2UlXFxcsGfPHgQEBJSofW5uLkaPHo2xY8ciNzeXna4IOTk5iIiIwJgxY0qcq8DAQOzevRvOzs5MIBFVOfeKdV1msVj3IBbrKoabm1uBt584fvx4sctGR0dj1apVTCIRERGRBatSvzA6duyIXbt2wcnJqUTtRVHEG2+8ga5du5Z4BFl1Ehsbi9DQUHz22WclXsbT0xM///wzGjduXP4/ClUq9Gqoxue9tGjtqXzUXXIWMPuQCR/9asSdbBZyiaji5UWfQMrrryJnz07lO/ga7nD433Q4vPgKBAeHKp+7mjVr4sCBA6Xa/yxbtgxt2rTBmTNn2PkKcPr0aYSGhmLFihUlXqZp06Y4cOAA3N3dmUAiqlJYrCvidxmLdRVKEATUrl37occ//PBDREdHF7rc+fPn8eSTT0KSyvdC1YKm77x27Vq5vy4RERFRlTneq2pvqF27dti/f3+hU0YUZP/+/WjevDnWrl3LHgFAkiR8+umnaNWqFU6cOFHi5erVq4dffvkFgYGBFbq+LrYqvN5Bi+kdNahhhlm4/rgq46UdRuy9JHIUJhFVCPHOHaTN/xjp8+ZCTk1VFkylgr5LGJznfAxd0+bVKo+1a9fGoUOH0LJlyxIvEx0djeDgYMyfP5/f+f85Dpg3bx6Cg4MRExNT4uWCg4Nx8OBB1KpVi0kkoiqDxboiDjlYrKs0ffr0eeixlJQUdOvWDdu2bcOdO3fuP56cnIw5c+agbdu2SExMLPd1q1mz5kOPpaenY9q0aYiLi4MkSbynHhEREVERquQvjpYtW+LXX39FvXr1SrxMSkoKRowYgbCwMFy8eLHadog///wToaGhePXVV5GTk1Pi5Vq0aFHqnJtbcB0Bn/XSoqe/AEHhwLssI/BVlIjp+0xITOUVgURUPmRZRs4vB5H6+mQY/zymOJ7a2wdO77wP+5ER1fbEWY0aNbB//3489thjJV4mJycHr7zyCtq1a4dTp05V6z4ZExODdu3aYfLkyaWaOrRz587Yt29fqS6aIiKyZCzWFY7Fuso3bNiwAh9PSkrCk08+iRo1asDX1xeenp5wd3fH9OnTS3xfWqU8PDygL6BPfPjhh/Dz84NarYbBYIDJZOKGJCIiIipAlf3lERgYiCNHjqBNmzalWm7v3r1o1qwZZsyYgfT09GrTEZKSkvD888+jdevWOHasdCeO+/Tpg19++QV16tSp9Pdho1FhbLAGc8M0qOesfMrMv27LmLzThE2nReSJHIFBROZjunYVabNnInPpIsg52cqCabWw7T8ITrPmQONbr9rn1snJCbt378aIESNKtdwff/yBli1bYurUqUhLS6tWOUtNTcWUKVPQqlUrHD58uFTLjho1Crt27YKjoyM/2ERk9VisKxyLdZajbdu2eOqppwp9XpZlJCQk4O+//35oJgGdTof33nuv/PqJSoVu3bpxIxERERGVUZX+BeLu7o4DBw4UehVaYXJycvDee++hQYMGWLRoEYxGY5XNUWZmJj744AP4+/vjq6++giiKpVr+1VdfxdatW2EwGCzqfdV3FfBhNw2eba6GTq0sligDX58W8UqkEaductQdESkjm0zI/mkbUmdMhenCecXxNI2bwvn9j2DXfxBUAk8s3qPT6bB69WrMmjWrwHurFMZkMuHDDz+Ev78/Fi5cWOWvAjeZTPjyyy/h7++Pjz/+uFTvV6VSYfbs2Vi5ciW0Wi07HRFZ936DxbrCv+9ZrLNIy5YtQ6tWrUq1jJ2dHVasWIF27dqV67r973//K9XxFxERERH9q8r/ErGxscG6devw+eefl/qE0s2bNzFhwgT4+/tj0aJFyMvLqzJ5ycjIwNy5c1GvXj1Mmzat1CMKHBwcsGnTJnzyyScQLPQksVpQYUBjNT7rqUWL2sp/MFzPAGYeMOGroyak53LUHRGVnvHCeaTOmIqsbzYApbxI4kEqgwGGEaPh9PoMqD14D7HCvPnmm9i2bRtcXFxKtVxSUhImTpyIwMBArFy5ssoV74xGI5YvX46AgAC88MILuH37dqmWd3Fxwfbt2/HGG2+wkxGRVWOxrohjjWpWrNNoNNDpdBazPu7u7sX+Jv/9998xbdo02NvbF9lWEASEh4fj/PnzGDZsGFxdXUu1LqW9SLd9+/bYuHEjnJ2d+UEiIiIiKu1xuPzgXAlV2OHDhzFkyBAkJCSUaflatWph4sSJGD9+fIE3V7YGCQkJ+OKLL7Bs2TKkpKSUKUazZs2wadMmBAYGWtV7PxgnYnW0iNRc5bEc9cCYlmq091HzW4SIiiVnZyNr6xbkRG43Szxti1awjxgHwdGJyS2hy5cvY9CgQThx4kSZlq9Xrx4mTZqE8PBwODg4WG0e0tLSsGLFCixYsADx8fFlitGyZUts2bIFvr6+7FhEZPH2TLuG3LSHZ8vQOQjwD3OETwdDtS3U3f3fK5BuXH/4JIGDI2x794FNl24VXqibOHEiFi5cWGy7Ro0aITY2lh38/+Xk5GDnzp2IiorCzZs3kZSUBIPBgICAAAQEBCA4OBj+/v4Vvl4mkwlHjhzB+fPnkZOTA4PBgBo1aiAoKMhst9dwd3cv0cVHM2fOxNtvv83OQkRERBavWhXtgH9OVr300ktYvXp1mWPo9XoMGDAA4eHh6NKli8WONLvHaDRi+/btWLlyJXbs2FHqKTDvEQQBr776KmbPnl3gjaWtQUaejBV/ijgYb55pLh/xUGF8iAa17Dn1BxEVLO/0KWQuXQjp7h3FsQRXNxhGRUAX1JKJLcu2yMvDjBkz8Mknn0CSyrYfcHR0xKhRoxAREYHmzZtbzXs/efIkli9fjlWrVpX5nr2CIGDKlCmYNWuWRY1EICIqyoNFOxbr/vVg0a4yi3X3sGhHpcWiHREREVU11a5od8+WLVvw/PPP4+bNm4rieHl5YfDgwRgwYADatm1rMQU8k8mEAwcOYMuWLdiyZUupp716kJ+fH5YvX45OnTpVie1/5paERVEmXEtXHkunBp5ppkbvhgLUAot3RPQPKTUFmWtXIe/oYTPsrVXQd+wEw7CRUNnYMLkKHTp0CCNHjizzyPt7WrRogREjRmDAgAHw9va2uPeZmJiILVu2YM2aNYiOjlYUy9fXF2vWrEGHDh3YgYjIqtwr2rFY97B7RTtLKNbdw6IdlRaLdkRERFTVVNuiHQCkpKRg6tSpWLp0KcyRBg8PD4SFhaFbt254/PHH4enpWaHv5/Lly9i7dy/27NmDffv24e7du4pjarVavPbaa3jzzTdha2tbpba/UZSx6YyIH85JMJlh4J2PkwoTW6vRwI0nAYiqu9zff0XmmhWQs7IUx1J7esEwZjy09RswsWaUmZmJt956CwsWLCjzCPT/CgkJQd++fREWFobg4GCo1RU/fbIoioiKisKePXuwbds2HDt2THFMjUaDSZMm4e233y71/WyIiCzBz+/fgFeogcW6AqTOngldy2CLKNbdw6IdlRaLdkRERFTVVOui3T2///47Xn75ZbOc3Povb29vtGnTBiEhIWjatCmaNGmCunXrKo4rSRLi4+Nx+vRpnD59GkePHsXhw4cVjxp8UNeuXTF//nw0adKkSm//q2kyFkaZcDZJ+UdBBeCJAAFDmqphq+WoO6LqRrx5AxnLl8B0zgwnkTQa2PZ8Arb9B0Gl0TC55eTEiROYMGECjhw5YraYzs7OaNeuHdq0aYM2bdogKCgINWrUMPu63759G9HR0Th8+DAOHz6M3377rcz3qy3Io48+ioULF+KRRx5hRyEiqyWJMgQ1j8sLIosiVGrLukc3i3ZUWizaERERUVXDot29HyyyjA0bNmD69OlITEwst9exs7ODt7c3vL29UadOHbi5ucHV1RX29vbQ6/XQ6XSQZRl5eXnIy8tDWloa7ty5g+TkZFy7dg0JCQm4cuUKcnNzy20dGzdujI8++gi9evWqVtt/10UJ62JEZBmVx3OzBZ4L0aBVHV7NS1QtvkNEETl7diHr242AUfmXiKZhIOwjxkFduw6TW0H7gG+++QbTpk1DfHx8ubxGzZo10aRJE/j5+cHHxwfe3t5wd3eHq6srXF1dYWNjA51OB41GA5PJhLy8POTk5ODOnTu4c+cOkpKSkJiYiISEBFy+fBlnzpzBrVu3ymVd69Wrhzlz5uDpp59m5yAiogrFoh2VFot2REREVNWwaPeAnJwcLFu2DHPnzsXVq1er1Xtv2LAhZsyYgWeeeaZSpvWyBCk5MhYfM+HIVfN8LEK9VBjbSgNXW17dS1RVmS5fQsbyxRCvKL/gQ2VrC9sBg2HTrSdUKn5vVLTc3Fx8+eWX+PDDD80+et0a1KpVC1OnTsWECROgt5Bp0oiIqHph0Y5Ki0U7IiIiqmo4DOgBNjY2eOGFF3Dx4kV89dVX8Pf3r/LvOSgoCOvXr8fZs2cxfPjwaluwAwBnGxWmttdiRkcN3O2UxztyVcaLPxmx55II1seJqhY5NwdZmzYi9Z0ZZinYaZsHwXnuPNh278WCXSXR6/V49dVXERcXh08//RR16lSPkY6enp5YsGAB4uLiMGnSJBbsiIiIiIiIiIgqCYt2hdDr9ZgwYQLOnz+PH3/8EV27dq1SJ1HVajX69++PgwcP4sSJE3jmmWcgCOwO97SsI2BBLy16NxAgKNzs2SZgYZSIaXtNSEiRmFyiKiAv5iRSXn8N2dt/ABQW5FXOLrB/6VU4vvY6BGcXJtcC2NraYtKkSYiPj8e6desQEhJSJd9n69atsX79esTFxeGll16CjY0NNz4RERERERERUSXSMAVFU6lUeOKJJ/DEE0/g8uXLWLVqFdasWYOEhASrfD+BgYEIDw/H8OHDUbt2bW7gIthoVIhopUFnPwlfHRVx+a6yE/Pnk2W8tsuEAY0FDGyshk7NkTRE1kZKT0PWpo3IPXTALPH07TrAbkQ4BFs7JtcCabVaDBs2DMOGDcORI0ewYsUKfPPNN0hNTbXa9+Tk5IQhQ4Zg9OjRaN26NTcyEREREREREZEF4T3tykCWZfz222/YsmULvvvuOyQmJlr0+gYEBGDgwIEYOHAgWrZsyQ1YBqIkY9tfEjadFpErKo9Xyx54LliD5rU4upHIWuQePYzMVcsgZ2QojiXUqg37iPHQBgQysVYmOzsbW7duxbfffoudO3ciOzvb4tfZzs4OPXr0wODBg9GvXz+OqCMiIovFe9pRafGedkRERFTVcKRdGahUKrRv3x7t27fHp59+iujoaOzZswe7d+/Gr7/+ipycnEpdPwcHB3Tq1AlhYWHo1q0bAgICuNEUUgsq9G+kRntvAUuOmXD8urJa940M4O2DJnTxEzDiETUc9Bx1R2SpxKRbyFy5DMbTMWb4MlHDpltP2A16Giqtlsm1Qra2thg6dCiGDh2KrKwsREZGIjIyEnv27LGoi3h8fHwQFhaGnj17okePHrCz42hOIiIiIiIiIiJLx6KdGQQFBSEoKAhTpkxBXl4eTpw4gcOHD+PIkSM4efIkLly4AKPRWC6vrdfrERgYiKCgILRp0wZt2rRBs2bNoFaruWHKgbtBhTce0+JQvIjV0SJSFNZn912WcPSqhDGt1Ojgw21GZElkSULu/r3I/HodkJenfIdb3x+GiPHQeNVlcqsIOzu7+yPZAeD8+fM4dOgQDh8+jMOHD+Ps2bOoiAkNBEFAo0aN7h8HPPbYY2jQoAE3EBERERERERGRlWHRzsx0Oh1CQ0MRGhp6/zGj0Yjz58/jwoULSEhIQEJCAq5du4bk5GQkJyfj7t27yM7ORl5eHvLy8qBSqaDT6aDT6WBnZwcXFxe4ubnBzc0NdevWhY+PD7y9vREQEID69euzQFcJHvNVI7iOgJUnROyPkxTFSs8DPv1DxL7LEp4L1qCWA0fdEVU2U0I8MpYvhhgfpzyYXg+7fgNh0/MJqAROiVuVNWzYEA0bNsTYsWMBAJmZmYiNjcWZM2cQGxuLuLg4JCYmIiEhAUlJSZCkku8/BEGAu7s7fHx84OPjA19fXzRu3BhNmzZFo0aNYDAYuAGIiIiIiIiIiKwci3YVQKvVokmTJmjSpAmTUYUYdCq8EKpBFz8JX0WZcC1NWbyYmzJejjRiaDM1nggQoBFYvCOqaHJeHrK3/4Dsbd8DkqQ4nqZxU9iPfQ5qtxpMbnXcTxgMCAkJQUhIyMN9TZaRkpKCO3fuIDU19f6FO0ajEVqt9v7FO05OTnB1dYWzszNUKu4XiIiIiIiIiIiqMhbtiBRq5C5gXnctNseK+P6sBJOC8/xGCVhzUsTBeAkTQtQIqMFROUQVxRh7GhkrlkK6dVNxLJWDIwzDR0Hfpi0TSwX3EZUKLi4ucHFxYTKIiIiIiIiIiAgAi3ZEZqFVqzC0mQYdfWQsjDIhNknZPYwSU2VM32tC74YChjRTw07L0RVE5UXKzEDW5m+Qu2+PWeLp2rSFYeRoCAZ7JpeIiIiIiIiIiIhKjEU7IjPydFRhdhctdl0Use6kiExj2WPJALafl/D7lX/udRfsyVF3ROaWdzwKGSuWQk5PUxxLqOkB+9FjoW3clIklIiIiIiIiIiKiUmPRjqgcdPdXI9RLwLLjIn6/ouy+WHeygfd/MaG1pwpjW2ngZsdRd0RKicm3kblqOYwnTygPJgiw6dINdk8/A5VOx+QSERERERERERFRmbBoR1ROnG1UeK2dBieuS1h8zIRbmcriHb0mI+amESOD1AirL0BQsXhHVFqyJCH354PIXL8ayM1VHE/tWw/2EeOh8fFlcomIiIiIiIiIiEgRFu2IylmL2gIW9NRifYyIHRckSApud5djAhYfE7H/soQJrdXwdeaUmUQlZbp6BZnLF8N06aLyYDodbPv0g22fflAJ/BwSERERERERERGRcizaEVUAvUaF0S016FxPwldHRVy6KyuKd+GOjNd2mTCgkYCBjdXQazjqjqgwstGI7MjtyP5+MyCKiuNpmzaHIXwM1O41mVwiIiIiIiIiIiIyGxbtiCpQPRcBc7upsO2chE1nROSYyh5LkoHNsRJ+SZAwPkSDoFoc7UP0IONf55CxfDGkG9cVx1LZ28Nu8FDYdO7CxBIREREREREREZHZsWhHVMEElQr9GqnR3lvAwigTTtxQNuruZiYw66AJj9cTMCJIDUc9R90RSdlZyN7yLXJ2R5olni64NQzhYyA4ODK5REREREREREREVC5YtCOqJDUMKrzZSYvfr0hYftyEuznK4u2Pk3D0moSIlmo85qtmgqnayjt5AhnLl0BOuas4luBWA4bwsdA1f4SJJSIiIiIiIiIionLFoh1RJWtbV8AjHlqsjhax97KkKFZGHrDgsIj9l/+ZMrOOA0fdUfUhpdxF5uoVyDsepTyYSgV95y4wDH0WKr0Nk0tERERkQRISEtC5c2cmgpCamsokEBERUZXCoh2RBTDoVJjYWoPH60lYGCXiSpqyKTNP3ZLxSqQRQ5qp0SdAgEZg8Y6qLlmWkfvbL8hauxJydrbieOq63rCPGA+NX30ml4iIiMgCZWVl4eDBg0wEEREREVU5LNoRWZBAdwGf9FBh8xkR35+VYFQw8M4oAWtPijgYJ2FiazUCaghMMFU54vW/kbF8CUznzykPptXCtndf2D45ACo1p5glIiIiIiIiIiKiisWiHZGlfSgFFYY006Cjr4xFUSacvqVs1N2VNBnT9prQq4GAoc3UMOg46o6sn2wyIWd3JLI2fwOYTMo/d4GNYT96LNS1ajO5REREREREREREVClYtCOyUHUcVJj1uBZ7LolYe1JERp6yeDsuSPjjioTxwRq09uKoO7JexksXkLlsMcRrVxXHUtnZwW7Q07Dp2p2JJSIiIiIiIiIiokrFoh2RhQurr0ZrTwHL/hTxW6KkKNbdHOCDX00IqaPC2FYa1DBw1B1ZDzknB1lbtyAncjsgy4rjaYNawj5iHAQnZyaXiIiIiIiIiIiIKh2LdkRWwMlGhcltNejiJ2FxlAk3M5XFi/pbxqlbRgx/RI3u/gIEFYt3ZNmMZ04hY+kiSHeSFccSXFxhGBUBXYtWTCwRERERERERERFZDBbtiKxIUC0B83tqseGUiJ/OS5AUDDbKMQFLj4s4ECdhQoga9Vw4ZSZZHiktFZlrVyHvyB9miafv2BmGYSOgsrVlcomIiIiIiIiIiMiisGhHZGX0GhXCW2jQuZ6Er46KuHhH2TSBF+/ImLLbhP6NBAxqrIZew1F3ZBly//gNmWtWQM7MVBxLXccThohx0DYIYGKJiIiIiIiIiIjIIrFoR2SlfJ0FfBCmwvbzEr4+JSLHVPZYkgxsiZXwS4KE8cEatKjNUXdUecSbN5CxchlMsaeVB1OrYdPzCdgNGAyVhrs8IiIiIiIiIiIislw8g0lkxQSVCn0D1GhbV8DiKBOOX1c26u5WJvDuIRM6+QoYEaSGsw1H3VHFkSUJOXt3IeubDYDRqHwH16AhDKPHQePpxeQSERERWYEWLVpgwIABTASZXePGjZkEIiIisgoqWZZlpoGoajh8RcLS4ybczVEey14HjG6hRqd6aiaWyp0pPg4ZyxZBTExQvmOzsYXtgEGw6d4LKhULz0RERERERERERGQdWLQjqmKyjDJWR4vYe0mCOT7cTWqqMCFYgzqOLH6Q+cm5ucj+cSuyf9wKmGF3pG3WHIaI56B2dWVyiYiIiIiIiIiIyKqwaEdURf11W8LCKBGJqco/4hoBeLqpGn0DBGjVLN6ReeSdjkHmiqWQbicp35k5OcEwYjT0IaFMLBEREREREREREVklFu2IqjCTJOO7WAlbYkUYJeXxvByBCSEaNHIXmFwqMyk9HVnfbkTuwf1miadr1wGGZ0dBMBiYXCIiIiIiIiIiIrJaLNoRVQPX02UsOmbCqZvm+bj39BfwTHM1DDqOuqPSyY06gsxVyyCnpyuOJdSqDfvwsdA24k3liYiIiIiIiIiIyPqxaEdUjey9JGLNSREZecpjOdsA44I1aOPFUXdUPPF2EjJXLoPx1EnlwQQBNt16wm7wEKi0WiaXiIiIiIiIiIiIqgQW7YiqmbRcGcuOi/g1UTJLvFZ1VBjXSgN3A0fd0cNkSULugX3I3LgOyMtVHE/jVx+GiPHQ1PVmcomIiIiIiIiIiKhKYdGOqJo6eUPComMm3MxQHstGAzzbXI3u/gLUAot39A9TYgIyli+GGHdZeTCdHnb9BsCmVx+oBI7uJCIiIiIiIiIioqqHRTuiaixPlLHxlIjtf0kQzfBN4OeiwsTWavi5sKhSncl5ecj+aRuyf/gOkJSP6NQ2D4Jh1Bioa9RgcomIiIiIiIiIiKjKYtGOiJCQImFhlIjzycq/DgQV8GSggKeaqKHXcNRddWM8ewYZK5ZCunlD+Q7KwQF2Tw+DTcdOTCwRERERERERERFVeSzaEREAQJJl7DgvYeMpEdkm5fHcDcD4YA1a1uaou2rRfzIzkbX5G+Tu222WeLrQR2EYMRqCgwOTS0RERERERERERNUCi3ZElE9ylozFx0w49rd5vho6+ggY1UINZxuOuquq8k4cR8aKJZBTUxXHEtxrwhA+FrqmzZhYIiIiIiIiIiIiqlZYtCOiAh25KmHpcRPuZCuPZdAC4S3U6FxPgErF4l1VId65g8zVy2E8cdwMeyMVbLp2g93Tw6DS6ZhcIiIiIiIiIiIiqnZYtCOiQmUbZayOFrHnkgRzfFE0dldhQogGno4s3FkzWZaR+/MBZK5fA+TkKI6n9vGFfcR4aHzrMblERERERERERERUbbFoR0TFOp8sYeFREQmpyr8uNALwVBM1ngwUoFWzeGdtTNeuInP5EpgunlceTKuFbZ9+sO3bHyqB9z4kIiIiIiIiIiKi6o1FOyIqEVGS8d1ZCVtiReSJyuN5OgATQjRoXJPFGmsgm0zIidyOrO++BUTlHUDTuCnsR4+FuqYHk0tEREREREREREQEFu2IqJRupMtYfMyEkzfN89XR3V/As83VMOg46s5SGS/8hczlSyD+fU35TsdggN3gobB5vCsTS0RERERERERERPQfLNoRUZnsvyxidbSI9DzlsZz0wLhgDR6ty1F3lkTOzkbWd98iZ9cOs8TTtQqBIXwMBEcnJpeIiIiIiIiIiIjoASzaEVGZpefKWP6niJ8TJLPEa1lbhXHBGtQ0cNRdZcuLOYnM5Ysg3b2rOJbg5gbDyDHQBbVgYomIiIiIiIiIiIgKwaIdESkWc1PC4igTrmcoj6VXA8Oaq9GzgQC1wOJdRZNSUpC5diXyoo6YYQ+jgr7T4zAMHQ6VjQ2TS0RERERERERERFQEFu2IyCzyRBnfnBax7ZwE0QzfKn4uKkwIUaO+K6fMrCi5v/6MzHWrIGdlKY6l9vSCYcx4aOs3YGKJiIiIiIiIiIiISoBFOyIyq8RUCV8dFXE+WflXi6AC+gQIeLqpGjYajrorL+KN68hYsRSmc7HKg2k0sO3dF7ZPDoBKo2FyiYiIiIiIiIiUdVHjAAAgAElEQVSIiEqIRTsiMjtZlrHjgoSNp0RkGZXHc7cDxgVr0KoOR92ZdTuJInJ2RSJryzeAUfmG0gQEwn70OKhr12FyiYiIiIiIiIiIiEqJRTsiKjd3smUsPmZC1DXzfM108BEwKkgNF1uOulPKdPkSMpYvhnglUfmOxNYWtgOfhk1Yd6hU3DZEREREREREREREZcGiHRGVu6PXJCw9bkKy8lulwU4LhLdQ4/F6AgtEZSDn5iBr63fI2fEjYIavf+0jLWAfMQ6CswuTS0RERERERERERKQAi3ZEVCGyjTLWnhSx66IEc3zpBNZQYWKIBl5OLNyVVF5MNDJXLoOUfFv5zsPZBfYjR0PXKoSJJSIiIiIiIiIiIjIDFu2IqEJdSJawMEpEfIryrx6NAAxqrEb/RgK0ahbvCiOlpSHrm/XI/eWQWeLpOzwGu2dHQrC1Y3KJiIiIiIiIiIiIzIRFOyKqcKIkY+s5Cd+eEZEnKo9XxwGYEKJBk5oCk/uA3MO/I3PNCsgZGYpjCbXrwH70OGgDAplYIiIiIiIiIiIiIjNj0Y6IKs3NDBmLj5kQfcM8X0Nh9QUMf0QNex1H3YlJt5C5YimMZ04pD6ZWw6ZHb9gNfAoqjYYdl4iIiIiIiIiIiKgcsGhHRJXuQJyINdEiUnOVx3LSA2NaqdHOW10tcylLEnL27kbWpg1AXp7ieBr/BjCMHgeNV112VCIiIiIiIiIiIqJyxKIdEVmE9FwZK0+IOBgvmSVei1oqjA/WoKZ99Rl1Z0qIR8byxRDj45QHs7GBXb9BsOnZGyoVRy4SERERERERERERlTcW7YjIopy+KWHhMROupyuPpVMDzzRXo3cDAWqh6hae5Lw8ZP+4Fdk/bgUk5UVPbdPmMIwZD7WrGzskERERERERERERUQVh0Y6ILI5RlPHNaRHb/pJgMsPAO19nFSa2VsPfVah6uTpzChkrl0G6dVP5DsHRCYbho6APfZSdkIiIiIiIiIiIiKiCsWhHRBbrSqqMhVEmnLut/GtKBaBPgICnm6phq7X+UXdSRgayvt2I3AP7zBJP92g7GEaEQzDYs+MRERERERERERERVQIW7YjIosmyjJ0XJayPEZFlVB7PzQ4Y30qDYE/rHXWXe+woMlcug5yepjiWUNMD9qPHQdu4CTsbERERERERERERUSVi0Y6IrMLdbBlLjplw5Jp5vrLaeQsIb6GGq631jLoTk28jc9VyGE+eUB5MEGAT1gN2g4dApdOxgxERERERERERERFVMhbtiMiqHLsmYclxE25nKY9lpwVGBqnR1U+ASmW5xTtZkpB7cD8yN64FcnMVx1PX84N9xHhovH3YoYiIiIiIiIiIiIgsBIt2RGR1so0y1seI2HlRgmSGb7DAGipMCNGgrpPlFe5MV68gc9limC5fVB5Mp4PdkwNg07svVILAjkRERERERERERERkQVi0IyKrdemOhK+OiohLUf41plYBAxsLGNhYDa268ot3stGI7J+2IfuH7wBRVBxP26w5DKPGQO1ekx2HiIiIiIiIiIiIyAKxaEdEVk2UZPxwTsKmM+L/sXffgVHU+f/HXzOzm06AQOi9I10CWBGk2uiiwEkVFNQ7G2Ivd6fYuyK9iwUUUAHpYENCJ1RpgYRi6Mmm7e7s7w9/X+88kZRJIOX5+I/NzHsn789k2N3Xfj6jTOfZliqWkO6NcalJ+cs3E827Z5dSJk+QffyY84t8RAmF3dFPITfcyMkCAAAAAAAAAAUYoR2AIuHXlIDGb/Bp8/G8uaR1rGXqrmaWSgRfull3dmqq0uZ9qvRl3+ZJvaDWVyl80FCZJSI5QQAAAAAAAACggCO0A1CkrDnk17TNfp3LcF4rMlgadqWl66tb+X7cmVs2KWXyBAXOnXVcyywbrfDBdyuoaTNOCAAAAAAAAAAoJAjtABQ5KZkBTd3s16qDdp7Ua1bB0D0xLlWIyPtZd/aZ0/LMmKrMjbF5cEU3FHxjJ4XfOUBGcDAnAgAAAAAAAAAUIoR2AIqsHb/aGhfr09Fk57WCLKlfE0u31jNlmc7Du0AgoIzv1ih19nQF0tIc17OqVlPEsHvkqlWbgQcAAAAAAACAQojQDkCR5vUH9PkOv+bvtuXLg4l3NUoZGtnKUt0yZq5r+I8mKmXKRPn27nZ+QG63Qm/trtBuPWVYFgMOAAAAAAAAAIUUoR2AYiHhfEDjYn3aleT8kmdIuqWeqX5NLIW6sz/rLuDzKX3JN0r94nPJ53N8HK6GVyhi6AhZ5SswwAAAAAAAAABQyBHaASg2AoGAlu63NXOrX6le5/XKhEr3tHIpplLWs+68+36RZ/J4+RMTnF+4w8IVdvsdCunQmUEFAAAAAAAAgCKC0A5AsXM2PaCJG3z6KSFvLn9tqhga3tKlqNA/z7oLpKcr9cvPlb5kkZQHl1v3lTGKGHK3zJKlGEgAAAAAAAAAKEII7QAUWxuP2pqwwaekVOe1Ql3SwOaWOtc2ZRi/hXeZ27fKM3mC7NOnHNc3S0cpfPAwBbVoycABAAAAAAAAQBFEaAegWEv3BTR7m1+Lf7Fl58HVsF4ZQ6MapKj0whnK/PmnPDnG4HY3KrzfXTJCQxkwAAAAAAAAACiiCO0AQNL+07bGxfp14IyzS2KbxB/Vb+cMhXudT9+zKldR+NARctetxwABAAAAAAAAQBFHaAcA/5/fDuirPbY+jfMrw5+zfaM9J3RX3BQ1PLXL+YFYlkJv6abQHr1luFwMDAAAAAAAAAAUA4R2APA/fvUENGGDT5uOZX15NG2/boxfpp57PleQ7XX83K569RUxdISsSpUZCAAAAAAAAAAoRgjtAOAvfBfv19TNfp1Nv/DPq587qIHbJqta8mHnF+OQUIX26auQTl1lGAbNBwAAAAAAAIBihtAOAC7CkxnQtC1+rThg//5YkD9Dt/6yQF0OfCNTzi+h7mbNFTF0hMzSUTQcAAAAAAAAAIopQjsAyIZdSbY+jPWp5P7t+lvcVEWnnXR+AS5ZSuEDhyi4VRsaDAAAAAAAAADFHKEdAGSDnZyslE9my/vdase1ApJ+qnydvr/qLg27roTqlTFpMAAAAAAAAAAUc4R2AJCFjPXr5Jk2WYGUZMe1jodX0IwmQ/VLVIPfLsKSbq5rqn9TS6Fu7mUHAAAAAAAAAMUVoR0A/AV/0q/yTJss7/atjmv5DEvLa3TRgnq95bPcf/p5VKg0Isal1pWZdQcAAAAAAAAAxRGhHQD8j4BtK2PlMnk++VjKzHBc70DJWprRZJgSI6tmuW3ryoaGt3SpTBiz7gAAAAAAAACgOCG0A4D/4jscr5TJ4+U/eMBxrXQrWF/V7allNbsqYGR/Bl2ISxrYzFLnOqZMg/AOAAAAAAAAAIoDQjsAkBTIzFTa1wuUtvBLybYd19se3UyzGg/S6dCyua5RN8rQqNaWqpdiyUwAAAAAAAAAKOoI7QAUe95dO5QyZaLsE8edX1RLRCq1W3+9o2u1/4zzy6tpSD0bmupzhaVgF7PuAAAAAAAAAKCoIrQDUGzZnhSlfv6JMlYuz5N6QVdfq/C7hsiMiJAdCOjrPbY+ifMr3ee8dvkI6d4Yl5pVYNYdAAAAAAAAABRFhHYAiqXMjbFKmTpJgfPnHNcyo8spfMhwBTVu8qefnfQENH6jTxuP5s2ltl0NU4NbWIoMZtYdAAAAAAAAABQlhHYAihX/6VPyTJss75ZNeXAFNRTSqavC+vaTERR00U2/j/dr6ma/zqQ7f9qIIGloC0vtaloMKAAAAAAAAAAUEYR2AIqFQCCgjNUr5ZkzU0p3npxZ1WsoYtg9ctWome19PJkBTd/i1/IDdp78Tk3KGbq3lUsVSzDrDgAAAAAAAAAKO0I7AEWeLzFBnskT5Nu313kxt1uh3Xsp9NbuMszc3V9uV5KtcbE+JZzPg8MxpTsaW+rWwJTLJLwDAAAAAAAAgMKK0A5AkRXw+ZT2zUKlzZ8n+f2O67kbNVb4kOGyypV3XMtnBzRvp615O/3y5cHEu2olDY1sZal+WZOBBwAAAAAAAIBCiNAOQJHk3btHnikT5D+a6PxCGRGhsL79FdLuxjw/zqPnA/pog09xv+bNpfimuqYGNLUU5mbWHQAAAAAAAAAUJoR2AIoUOy1VaXM/U/qyJXlSL6hVG4UPGiYzMjJfj3v5fr9mbPUrJdN5rdIh0ogYl9pUYdYdAAAAAAAAABQWhHYAiozMLZvlmTpB9pkzjmuZZcoofPDdCmrW4pId/7n0gCZv8uv7w3ae1GtVydDwGJfKhjHrDgAAAAAAAAAKOkI7AIWeffasPDOmKHPD+jy4KhoKvrGjwu8cICM45LL8PluO2Rq/wacTHue1QlzS35pZ6lrHlGkQ3gEAAAAAAABAQUVoB6DQCgQCyvh+jVJnz1AgNdVxPatKVYUPu0fu2nUu+++W4Qvokzi/vtpjy86Dq3SdKEMjW1mqWZolMwEAAAAAAACgICK0A1Ao+Y8dVcrUifLt3uW8mMul0Fu7K7RbTxkuV4H6PQ+esTUu1q99p51fqk1D6t7AVN9GloJdzLoDAAAAAAAAgIKE0A5AoRLw+ZT+7SKlfvG55PU6rudq0FARQ4bLqlipwP7OdiCgb/bamrPdr3Sf83rlw6V7YlxqXpFZdwAAAAAAAABQUBDaASg0vPv3yTN5vPwJR5xf/MLCFNbnTgV36CSjkNzr7WRqQOM3+LTxaN5ctm+oYWpwc0slQ5h1BwAAAAAAAACXG6EdgAIvkJGu1C/mKn3JN1IeXLLcLVoqYshwmaVKFcp+rDtia+JGn86kO68VESQNaWGpfU2LEw0AAAAAAAAALiNCOwAFWubWzfJMmyz71EnHtczSpRU+cKiCWrYq9H1J9QY0Y4tfS/fbeVKvUTlDI2NcqhTJrDsAAAAAAAAAuBwI7QAUSPb580qdM1MZP3yXJ/WCb2ivsP53yQwNK1J92nPS1rhYvw6fc34pd5lS30aWejQ05TIJ7wAAAAAAAADgUiK0A1DgZPz4vTyzpimQkuK4llWpssKHjpC7Xv0i2y+fHdCXu2zN3eGXNw8m3lWJlEa2cqlhtMnJCAAAAAAAAACXCKEdgALD/+sJeaZOlHdHnPNilqXQm25VaK/bZbhcxaJ/x5ID+ijWp+2/5s1lvWsdUwOaWgoPYtYdAAAAAAAAAOQ3QjsAl13AtpW+bIlSP/9Eysx0XM9Vp57Ch42Qq3KVYtnPFQf8mr7FrxTnrVTpEGl4S5euqsqsOwAAAAAAAADIT4R2AC4r36GDSpk8Xv74Q86LhYQorFdfhXS5SYZRvGeHnc8IaMomv9bG23lSr2UlQyNauhQdzqw7AAAAAAAAAMgPhHYALotAZqbSFnyhtK8XSHlwGXI3aabwYSNkRZWhuf9l63FbH23w6YTz2wMqxCX1b2rppjqmLJPwDgAAAAAAAADyEqEdgEvOu2O7UqZMlJ30q/OLWGRJhQ8couDWV9HYv5DhC+jTOL8W7rFl58EVv3ZpQyNbW6pVmiUzAQAAkHdmz56tVatW0QjkuW7duqlbt240AgAAFHguWgDgUrFTUpT66WxlrMmbN+LB116vsL8NkhkeQXMv1ieXoYHNXbqhhq0P1/v1y2lnyd3+MwE9ttSnbvVN3dHYUrCLWXcAAABw7ocfftDkyZNpBPJclSpVCO0AAEChQGgH4JLIWL9OnulTFEg+77iWWb6CIoYOl7thIxqbA9VLmRrbydCiX2x9vM2vdF/ua9kBaf5uWz8csXVPjEtXVmTWHQAAAAAAAAA4QWgHIF/5T56UZ/okebducV7MNBXS5WaF9e4rIyiI5uamhYahW+tZurqKqQkbfIo96mzWXZJH+vcan9pWNzW4haVSIcy6AwAAAAAAAIDcILQDkC8Ctq2Mlcvl+XS2lJHhuJ5Vs5Yiht0jV7XqNDcPlAkz9ERbt35OsDVxo0+n05zVWxtva+NRW4NbWLqxpinDILwDAAAAAAAAgJwgtAOQ53xHDsszeYJ8B/Y5LxYUrLCevRVy060yTJZgzGttqphqWt6tmVv9+nafLSfz7jxe6YP1fq06aGtkK5cqRxLcAQAAAAAAAEB2EdoByDMBr1dpXy9Q2sIvJb/fcT13k6YKH3y3rOhyNDcfhboNjYhxqV1NWx+u9+vwOWdLZu5MCuihJV7d3shSjwam3BbhHQAAAAAAAABkhdAOQJ7w7t6llCkTZB8/5riWEVFCYf0GKOT6djT2EqpXxtQbXQx9ucvW3J1+ZTrIXX22NGe7X2vj/RoZ49IV5ZglCQAAAAAAAAAXQ2gHwBE7NVWpn89RxopleVIvqM3VCh84VGaJEjT3MrBMQ30aWbqumqmPNvi07YSzWXeJ56WnV/rUubapu5pZCg9i1h0AAAAAAAAAXAihHYBcy9y0QSlTJylw7qzjWmbZaIUPuVtBTZrR2AKgQglDz7d3a9VBv6Zt9is501m9pfttrU+0dfeVLl1TjVl3AAAAAAAAAPC/CO0A5Jh95rRSpk+Rd9MG58UMQyEduyisbz8ZwcE0t4BpX9NSTCVTUzb7teaQ7ajW2XTp9R99uvLgb/fQKxfOrDsAAADkXHBwsBo0aEAjoLi4OPnz4H7qAAAABQWhHYBsCwQCylizSqkfz1QgPc1xPatadUUMu0eumrVobgFWItjQP65yqX1NWx/F+nQ8xVm9TccC+scir/o3tXRzXVOWSXgHAACA7KtVq5a2bNlCI6Do6GidPHmSRgAAgCKD0A5AtviPJiplygT59u5xXsztVmi3ngq9tbsMy6K5hUTT8qbevsmtT+P8Wrjblt/B7e4y/NLU/z97b2QrS7WjWDITAAAAAAAAQPFGaAfgogI+n9IWf620L+dKPp/zi07DRooYOlxW+Qo0txAKsgzd1cylttVtjYv1a++pgKN6B84ENGaZT7fVN3VHY0shLmbdAQAAAAAAACieCO0A/CXvL3vlmTJB/sQEx7WMsHCF3dFPIe070tgioHopU2M7Glq8z9bsrX6lOchz7YC0YLetHw/bGhHjUstKzLoDAAAAAAAAUPwQ2gH4k0BamlK/+EzpS5dIgYDjekExrRU+aKjMkqVobhFiGIZurmvpqiqmJm7w6edEZ+dKUqr04lqfrqtmauiVlkqFMOsOAAAAAAAAQPFBaAfgDzI3b5Rn2mTZZ047rmVGlVH4oKEKatGSxhZhUaGGxlzv1vpEWxM3+HQqzVm97w/b2nTM1uDmljrUMmUYhHcAAAAAAAAAij5COwCSJPvcWXk+nqnMn35wXswwFNy+g8LvGCAjNJTmFhOtK5tqUs6tWdv8WvKLLSfz7lK90oexfq06ZGtkjEtVShLcAQAAAAAAACjaCO0AKP37tUqdNV2BVI/jWlblKgofdo/cderS2GIo1G1oeEuX2tWwNS7Wr0NnnS2ZuSspoIe/9ar3FZZ6NTTltgjvAAAAAAAAABRNhHZAMeY/cVwpUybIt2tnHlxNXAq9tbtCu/WU4eLSUtzVLWPqtc6GFuy29dkOvzL9ua/ls6VP4/z6Lt6vka1calTOpMEAAAAAAAAAihw+WQeKoYDfr/RvFyl13meS1+v8QlKvviKGjpBVqTLNxe8s01CvKyxdU83U+A0+bT3ubNbd0WTpmZU+daxlamBzSxFBzLoDAAAAAAAAUHQQ2gHFjO/AfqVMHi//kcOOaxmhoQrtc4dCOnaRYRCg4MIqRBh6rp1bqw/6NW2LX+cznNVbfsBWbKKtYVdauq66RYMBAAAAAAAAFAmEdkAxEcjIUOqXc5W++GspEHBcz938SkUMuVtm6Siai2xpV9NSy0qmpm72a/Uh21GtcxnSmz/5teqgrXtiXCoXQWgMAAAAAAAAoHAjtAOKgcxtW+WZNkn2ySTHtYySpRQ+cIiCW7WhscixEsGG/n6VSzfWtDVug0/Hkp3V23w8oL8v9qp/E0u31DNlmYR3AAAAAAAAAAonQjugCLOTzyt1zmxlfL8mT+oFt22nsP4DZYaF0Vw40ri8qbe7uvXZDr/m77LldzD5M9MvTdvi15pDtka2tlQnyqTBAAAAAAAAAAodQjugiMpY96M8M6YqkJLsuJZZoaIiho2Qu35DGos847YMDWjqUtvqAY2L9Wn3SWfLth48G9Djy3y6pa6pO5tYCnUz6w4AAAAAAABA4UFoBxQx/qRf5Zk2Sd7t25wXsyyF3HSrwnr2keF201zki6olDb3YwaVv99matc2vVG/ua9kB6au9tn5KsDWipUsxlZl1BwAAAAAAAKBwILQDioiAbSt9+bdK/ewTKTPD+cWhVh2F332PXFWq0lzkO8Mw1LWupdZVTE3a6NO6BGez7k6mSi9959M1VU0NvdJSVCiz7gAAAAAAAAAUbIR2QBHgOxyvlMnj5T94wHmx4GCF9e6rkM43yTCZpYRLKyrU0GPXubUh0daEjT6dTHVW78cjtrYctzWwmaVOtU0ZBuEdAAAAAAAAgIKJ0A4oxAKZmUpb+KXSvl4g2bbjeu5mLRQ+aJissmVpLi6rmMqmGpVz6+Ntfi3eZ8t2MPEu1St9tMGv1YdsjWzlUtWSBHcAAAAAAAAACh5CO6CQ8u7coZQpE2T/esJxLaNEpML736Xga6+nsSgwQt2GhrV06Yaatj5c79ehs86WzNx9MqBHvvWqZ0NTfa6w5LYI7wAAAAAAAAAUHIR2QCFje1KU+ukcZaxekSf1gq65TuF/GywzIoLmokCqE2Xqtc6GFu6x9WmcX5n+3Nfy2dLnO2x9f9jWyBiXGpdnCVgAAAAAAAAABQOhHVCIZMT+LM/0KQqcP+e4llmuvCKG3C13oyY0FgWeZRrq2dDStVVNjd/g0+bjzmbdHUuWnl3lU4dapgY2s1QimFl3AAAAAAAAAC4vQjugEPCfPiXPtMnybtnkvJhpKqTzTQrrc4eMoCCai0KlXIShZ9q5tfaQX1M3+3Uuw1m9FQdsxSbaGnalpeurWzQYAAAAAAAAwGVDaAcUYAHbVsaqFfJ8OltKT3dcz6peQxHD7pGrRk2ai0KtbQ1LV1YyNW2zXysP2o5qnc+Q3vrptzr3xrhUPoJZdwAAAAAAAAAuPUI7oIDyJRyRZ8oE+fb94rxYUJDCevRWyM23yTC5hxeKhoggQ/e3cal9TVvjYn06muys3tbjAf1jsVd3NrZ0W31Tlkl4BwAAAAAAAODSIbQDCpiAz6e0r+YrbeGXkt/vuJ67UROFDx0uK7oczUWR1Kicqbe6ujV3p19f7rLlczDxLtMvzdjq19p4WyNbWapbhpAbAAAUTZ/ueV0ty3dSnVLNaAYAAABQQBDaAQWId89upUyZIPvYUce1jIgIhd0xQCE3tKexKPLclqF+TVy6vlpA4zb4tCsp4KjeobMBPb7Mp5vrmerfxFKom1l3AACgaNl/dpvWHVuk2iWbqmvNIYR3AAAAQAFAaAcUAHZaqtI+/1Tpy7/Nk3pBba5W+F1DZEZG0lwUK1VKGvr3jS4t3W9r5la/Ur25rxWQ9M1eW+uO2Boe41Lrysy6AwAARc/+c9v0wZaHCO/wl2bOnKmEhIQ/PDZ69Gi5XHykBAAAkNd4hQVcZpmbNypl6iQFzp5xXMssU1bhQ+5WUNPmNBbFlmEY6lLHUuvKpiZv8uvHI7ajeqfSpJe/8+nqKoaGtXQpKpRZdwAAoOghvMNfuf/++3X+/Pk/PPbQQw8R2gEAAOQDXmEBl4l99ow8M6Yqc8N658UMQyEdOivsjn4ygkNoLiCpdKihR691aeNRWxM2+JSU6qzeTwkBbTnu1cDmljrXNmUYhHcAAKDoIbyDo/e5tq2UlJQ/PGaapiIiImgOAABANhDaAZdYIBBQxtrVSv14hgJpaY7rWVWrKWLYPXLVqk1zgQtoWcnUOze79fE2vxb9Yst2cLu7NJ80foNfqw7aGtnKUvVSLJkJAACKJsI75Mb27dvVvPkfV35p0KCBdu3aRXMAAACygdAOuIT8x44qZcoE+fbsdl7M7VbobT0UelsPGZZFc4GLCHEZGnqlSzfUsDUu1q8DZwKO6u09FdCj3/rUs6GpPo0sBVnMugMAAEUT4R0AAABw6RDaAZdAwOdT+uKvlfrlXMnnc/6H2+AKRQwdLqtCRZoL5EDtKFOvdDL09V5bn2z3K8Of+1r+gDR3p60fDtu6p5VLTcsz6w4AABRdhHcAAABA/iO0A/KZd/8v8kyeIH/CEce1jLAwhfXtr+D2HbifFpBLlmmoewNLV1c1NWGDT5uOOZt1dyxFen6VT+1rmhrc3FKJYP42AQBA0UV4BwAAAOQfQjsgnwTS05U67zOlL10sBQKO6wW1bKXwQcNklipFc4E8UC7c0NM3uPVdvF9TNvl1LsNZvVUHbW1ItDX0Sks31GDJWgAAULQR3gEAAAB5j9AOyAeZWzbLM32S7FOnHNcyS5dW+KBhCroyhsYC+eD66paurGhq+ha/lh+wHdVKzpTeWefXqoO27o1xqUIJZt0BAICi7ffwrlQzda0xmPAOAAAAcIDQDshD9vlz8nw8U5k/fp8n9YLbdVBYvwEyQ8NoLpCPwoMMjWrtUrsatsZt8CnxvLN6204E9OASr/o2stS9gSnLJLwDAABF2/6zW4tFeBcIBJSQkKBffvlFZ86cUZUqVVSrVi1FR0dnuW9ycrIOHjyohIQEpaWlqVy5cqpYsaJq1aol0+T+yAAAACC0A/JMxg/fyTNrmgIej+NaVqXKCh86Qu569WkscAldUc7Um13cmrfTry922fI5mHiX6ZdmbfPru3hbI1tbqleGD2IAAEDRV1jDu5SUFN16661/eKxz58568sknJUmnTp3S22+/rffff19nz5790/5VqlTR008/raFDh8rtdv/hZ4sWLdL48eO1ePFieb3eC+7br18/jRkzRmXKlLnoccbHx2vQoEF/eKx+/foaP358tn/XadOmadq0ab//24LTX+QAACAASURBVDRNvfHGG2rRokWuejd69GjFxsb+3sf/dfjwYbVr1+4Pj61YsUKWxZLyAAAA/4vQDnDI/+sJeaZOlHdHnPNilqXQW7optEdvGS7+PIHLwW0ZurOJS9dXD2hcrE87k5zdkzL+XEBPLPPpprqmBjS1FOpm1h0AACj6Clt45/P5tGbNmj88lpCQoCeffFJz587VkCFDLhhI/fe29957r959912tXLlS5cuX17lz53Tfffdp9uzZF33uhIQEvfbaa5ozZ47mzp2rNm3a/OW2qampfzrOXbt25Si0O3jw4J9qLFmyJNeh3datW/9UL6tjDuTBfd8BAACKIr72D+RSwLaVtvhrnX3i0TwJ7Fx16qnkv19RWJ87COyAAqBypKF/3ejSyFaWwt0OrxeSFv1i64FFXq1PsGkuAAAoNv4vvHt/y0Pad3ZroTv+8ePHq2/fvhcN7P7bzp07ddttt+n8+fPq2LFjloHdf0tISNBtt92mEydOcOIAAAAUUyQDQC74Dh1UyuTx8scfclzLCAlVaJ87FNKpiwyDGThAQWIYhjrVttSqsqnJm/z64bCzwO10mvTy9z61qWzo7pYulQnjbx4AABQPhXHZzP3792vUqFE5nhUWGxurUqVK5Wo2WVJSkh588EHNmTOn0Iwty1wCAADkHUI7IAcCGRlKnT9P6Yu+kvJgOQ93i5YKHzRMVlQUzQUKsFIhhh65xqX2NW2N3+BTksNbV/6cGNC2E179rZmlLnVMmQT2AAAUOcmZp/XWpvsK7PGdyzh5WZ63sIV3tv2fL22Fh4erX79+iomJUcmSJbV9+3Z98sknOnDgwJ/fO/7P+0W3260+ffqoRYsWqlKlihISEjRt2jTt3LnzT/vOnz9fKSkpioiIKBTn+jvvvKOlS5dK+m2pzEmTJv3h5+XLl9czzzzzh8cI+gAAAC6M0A7Ipsy47fJMnSg76VfHtYySJRU+YJCCr7qGxgKFyJUVTb17k1tztvv19V5btoPsPs0nTdzo15pDtka2slS9FCtWAwBQlPgDfp1JZ5nDv1LYwru2bdtq6tSpqlWr1u+P3Xnnnfr73/+ujh07Ki7ur2+Z0KZNG02cOFFNmjT5w+OPPPKIHnzwQb333nt/eDw9PV2LFy/W7bffXijGsl69eqpXr56kC4d2pUuX1n333cdJDwAAkA18QghkwU5OVsqkj5T86ot5EtgFX3+DSr38JoEdUEgFuwwNbuHSq51dql3a+Qy5vacCevRbn2Zt9SnDF6DBAACgWPnve95drtl/WWnfvr1Wr179h8Du/5QvX14TJ078y33r1q2rNWvW/CmwkyTTNPX666+rTp06f/rZ9u3bOTkAAACKIWbaXSJer1fx8fE6fPiw4uPjlZCQoFOnTun06dM6ffq00tPTlZmZqczMTBmGoaCgIAUFBSk0NFRRUVGKiopS2bJlVbVqVVWrVk3Vq1dXtWrVZJrkrvkpY92P8sycqkBysuNaZvkKihg6XO6GjWgsUATUKm3q5U6GvvnF1ifb/Ur35b6WPyB9scvWD0ds3RvjUrMKXNuLmjNnzujQoUOKj49XfHy8Tpw48ftrgPPnzysjI0OZmZny+XxyuVwKCgpScHCwIiMjFRUVpTJlyqh8+fK/vwaoUaOGSpcuTWMBAEXG/rNbFXdyT4E7LtM09eabb170/uOtW7dWVFSUTp8+/aefvffeewoODv7LfYOCgnTNNddo3759f3g8KSmJkwIAAKAYIrTLB8nJyYqNjdW6deu0bds27dixQ3v27JHX683T5wkJCVHDhg3VqFEjNWvWTFdddZVatmyp0NBQBsEh/8mT8kybJO+2LXnxLk8hXW9RWK/bZQQF0VygCLFMQ93qW7q6iqkJG33aeNTZTLkTKdILq31qV8PU4BaWIoO5111htH//fq1bt06xsbGKi4vTjh07dPz48Tx/nooVK6pRo0Zq3LixYmJidNVVV6l27doMAAAAeWjgwIFq3rx5Fm/5TFWuXPlPoV3r1q3VpUuXLJ/j/5aW/G+EdgAAAMUToV0eSE1N1Zo1a7Rs2TKtWLFCcXFxf7hZdX5JT0/X5s2btXnzZs2aNUvSbze3bt68uTp27KhOnTrp2muvVRBBUbYFbFvpK5Yq9bM5UkaG43pWzVqKGHaPXNWq01ygCIsON/RUW7d+OOzXlE1+nUl3Vm/1IVsbjtoa0sJS+5oWDS7gDhw4oKVLl2rZsmVau3atTp68NEt7HTt2TMeOHdPy5cv/cy5GR6tt27bq1KmTOnXqdMFlvAAAQPZ17do1W9tdaAZ8gwYNsrVvRETEBT9nAAAAQPFDaJdLp06d0oIFCzRv3jwtX75cmZmZBeK4vF6vYmNjFRsbq7Fjxyo8PFw33XSTevfurVtuuUUlSpRg8P6C78hheSaPl+/AfufFgoIV1ut2hXS9WQZLmALFxrXVLDWvYGrGVr+W7Xf25Y2UTOm9n/1afei3JTMrlmDWXUGyYcMGzZs3T1988YX27t1bYI4rKSlJ8+bN07x58yT99s393r17q1evXoqJiWHgAADIoWrVqmVru5CQkD89xpdnAAAAkFOEdjmQkZGhhQsXaurUqVq6dKn8fn+BP2aPx6O5c+dq7ty5Cg4OVrdu3TR48GB16dJFlsXsDUkKZGYq7av5Svt6gZQHY+pu0kzhQ+6WVTaa5gLFUHiQoZGtXGpXw9a4WJ8Szjurt/1EQA8u9qpvY0vdG5hymYR3l8vBgwc1ffp0TZ8+XYcOHSoUx7x3716NHTtWY8eOVc2aNTVo0CANGjRINWrUYEABAAWO2wopcMdUvXruV00htAMAAEBOMQUoGw4cOKCHH35YlSpVUt++fbV48eJCEdj9r4yMDH3++ee65ZZbVLVqVT333HP5co+dwsS7a6fOPj1GaQu+cBzYGSVKKGLEKEWOfoLADoAaRpt6o4tbdza25HL4v63XlmZv8+vRb33ac9KmuZeQbdv68ssvdeONN6p27dp64YUXCk1g978OHjyo559/XrVq1VKHDh00f/78S7KcNwAAWYkMKqOede5Xy3IdC9Rxud1uVahQIdf7h4WFMbgAAADIEUK7i/j+++/Vo0cP1a1bV2+99dafbipdmB07dkz//Oc/Vb16dd11113aunVrsRpb2+NRytSJOj/2n7KPH3NcL+jqa1Xq5TcVfF1b/nAA/M5tGerb2NLbXd1qVM75DLnD5wJ6crlPEzb4lOoN0OB85PF49NZbb6l27drq1auXVq1apUCgaPQ8EAho5cqV6tmzp+rUqaO3335bHo+HQQcAXHL/F9Y93Wa22lbpJcssWKvBlC5dWmYRvN1BQbm9BwAAAP6M5TEvYM2aNXrhhRe0atWqIv+7ZmZmatasWZo9e7a6d++uZ599Vi1atCjSv3PGhvXyTJ+iwLmzjmuZZaMVPnS4gho35Q8HwF+qFGnoXze6teKAX9O3+JXi4HOSgKQl+2z9nGBreIxLV1Xh+zd5KSUlRe+//77eeOMNnTx5ssj/vgcPHtRDDz2kl156SY8++qhGjRqliIgITgQAyAMhVphuqNKnwB7f+uNLlOZLuSzPHRlURh2q9dPVFW+V2wriZLnEzpw5QxMAAAAKKEK7/7JlyxY9+uijWrFiRb4+j2EYqlixoipVqqSoqChFRUWpRIkSCgoKUlBQkAKBgDIzM5WZmanz58/r9OnTOnXqlBITE/Xrr7/myzEFAgHNnz9f8+fPV+/evfXKK6+odu3aRWp8/adPyzNjirybNuTFICqkc1eF9blTRnAwfzwAsqVDLUsxlUxN2ezXd/HOliU8ky69+r1PrSobGt7SpbJh3OvOCa/Xq3Hjxumf//ynTp06la/PFRoaqmrVqik6OlpRUVEqXbq0QkJCFBwcLJfLJa/Xq8zMTKWnp+vMmTM6ffq0kpKSdPjwYaWlpeXLMSUlJWnMmDF67bXX9Nxzz+nee++Vy8XLRABwIsQVrh51RhXY49t5at0lD+0I6woGQjsAAICCi09jJB09elRPPfWUZsyYkaf3djEMQw0aNFCrVq3UuHFjNWrUSPXr11fVqlUVFJS7Nyjp6emKj4/X7t27tWPHDm3fvl3r16/XgQMH8uy4582bp6+++kqjRo3Sc889p1KlShXq8Q0EAspYvUKpc2YrkO78w06rWnVFDLtHrprcVBxAzpUMMfTQ1S61r2Hrow0+/epwVcLYxIC2n/BqQFNLN9U1ZRqEdzm1YMECjR49Wr/88kue1i1RooRatWql5s2bq1GjRmrUqJFq1aql6Ojc3/c0KSlJ+/fv186dOxUXF6ctW7YoNjZWKSl586HryZMn9cADD+i9997Ta6+9pm7dunGCAAAcI6wrWE6cOEETAAAACqhiHdrZtq0PP/xQTz31lM6fP++4nmEYatasmTp16qQOHTqoTZs2eR54hYSEqH79+qpfv766d+/+++NJSUn68ccftWLFCi1btky7d+929DyZmZl6++23NWfOHL311lvq169foRxjX2KCPFMmyPfLXufF3G6F9uit0Jtvk2FZXD0AONK8oql3bnLrkzi/vtpjy3Zwu7R0nzR5k19rDtka2cpSzdIsmZkd8fHxuu+++/TNN9/kSb3IyEi1a9dOnTt3Vtu2bdWoUaM8vw9OdHS0oqOjddVVV/3h9UxcXJzWrl2rZcuWadWqVUpOTnb0PHv37lX37t1122236f3331e1atU4YQAAOf+/kbDukvD7/Tl6rx8bG0vTAAAACqhiG9pt375dw4YNc/xi1eVyqV27durdu7d69OihChUqXJbfJzo6Wt27d/89yDt06JC++OILzZs3Tz/99JMCgdx9GnzixAn1799f06ZN0/jx41WjRo1CMb4Bn09p3yxU2vx5Ug7ewPzlOF/RWBFD7pZVvgJXDQB5JthlaFBzl9pWtzUu1q99pwOO6u07HdDopT51b2CqbyNLwS5m3V2Ibdt6++239eyzz8rjcTbVsUKFCurZs6d69eqldu3aXZYlJU3TVNOmTdW0aVPdf//98vl8Wr16tebNm6f58+fr+PHjua791VdfaeXKlfrXv/6lf/zjH3keQgIAiibCuvxjXGBVhVOnTunkyZMqW7ZslvuvW7dOqampNBIAAKCAKnafvNi2rddff12tWrVyFNg1btxYb7zxhhITE7Vs2TLde++9ly2wu5AaNWro4Ycf1g8//KCDBw/qhRdeUK1auV/OcenSpWrWrJmmT59e4MfY+8senXt6jNLmfeY4sDPCwxU+bIRKPv40gR2AfFOztKmXO7k0tIWlEIeZjx2Qvtxl68HFXm05ZtPc/xEfH6/27dvrkUceyXVgFxISojvuuENLlixRYmKiPvzwQ3Xs2LHA3APO5XKpY8eOGjdunBISErR48WL17dtXwbm8B6vH49HDDz+sDh066PDhw5xEAIC/FBlURj3r3K+n28xW2yq9COzyQZkyZS74+KZNm7K1f2F4Tw8AAFCcFavQLjExUR06dNDo0aOVkZGR4/1dLpduv/12ff/999q+fbsefvhhlStXrsD/3tWrV9ezzz6rffv2afHixeratesFv52XlfPnz2vw4MHq06ePzp49W+B+z0Bamjwzp+r8v56T/2ii43pBra9SqVfeVMgNN3KlAJD//yEbhm6tb+ndm91qVcn5DLkTHumfa3x6+yefzqUHaLCkWbNmqWnTplq7dm2u9q9SpYrGjh2rhIQEffLJJ+rSpUuBn3lmWZa6du2qTz/9VImJiXrxxRdVuXLlXNVavXq1mjZtqo8//piTCQDwB4R1l06ZMmUu+EWhjRs3Zrnvli1bNG3aNJoIAABQgBWb0G7lypVq0aKFVq9eneN9g4ODNWrUKO3fv1+fffaZrr322kLZA8Mw1LVrVy1evFg7duzQgAEDZOXi3mzz5s1Ty5YttWXLlgLzu2Vu3qizjz+i9GXfOv+jiCqjEg8/phL3PygzsiRXCQCXVNkwQ0+0dWv0tS6VDnFeb228rQcWebXygL/Y9jQzM1MjR47UXXfdlat72NavX18zZ87UwYMH9fjjj//lN9wLujJlyujJJ5/UoUOHNH36dNWrVy/HNc6dO6cBAwbovvvuU2ZmJn+wAFDMEdZdeqZpqmLFin96/NVXX73oe/T/u1+tbefvSgwX+oJwYmJivj8vAABAkXm9Vxx+yZdfflmdO3dWUlJSjvZzuVy/h3UffPCBqlWrVmR60rBhQ82aNUu7du1S//79czzz7sCBA7r66qsv+7f07HNnlfzBO0p+6zXZZ047fXeh4A6dVerlNxTU/EquDgAuq6urmnrvFre61HH+X3VKpvT+er+eXenV0eTiNevuyJEjuu666/TRRx/leN/atWtr9uzZ2rlzp/72t78VmOUvnXK5XBo4cKB27typWbNm5Wr57A8//FDXX3+9EhIS+GMFgGKIsO7yuu222/702NmzZ9W5c2ctXLhQp0//573xqVOnNHbsWF1zzTWXZJnrC61GlJycrCeeeEIHDx6UbdvcUw8AAOAiinRol5mZqYEDB+qJJ56QP4f3NuvWrZvi4uL0wQcf5HoZqcKgbt26mj17ttavX6/rr78+R/ump6dryJAhGjNmjAKBS/8hcPra1To75hFl/vyT41pW5SqKfOafihg0VEZICFcGAAVCmNvQPTEuje3oUrWSzpfMjPs1oAcXe/X5Dr989qW/bp/0XNrn3LBhg1q3bp3je9iWLl1ab775pnbu3Kn+/fsX+CUwc/1/n2VpwIAB2rVrl15//XWVKlUqR/uvX79erVu3zvY9dAAAhV9kUBRhXQEwYMCACz6elJSk7t27q2zZsqpRo4YqV66s6OhoPfnkkzp16tQlObby5ctf8D66r776qmrVqiXLshQeHi6fz8dAAgAAXECRDe1Onz6tzp07a+bMmTnar2bNmlq8eLEWLFig+vXrF5sTISYmRmvXrtWsWbNyfJ++V199VX379lVaWtolOVb/8WM6N/af8kz6SIFUj7NiLpdCe/ZRyX+9LHedulwRABRI9cuaer2LS/2bWHI7/J/bZ0tztvv1yBKfdiddumWKMnwBPbPSq12X6DkXLFigG264QcePH8/RfgMHDtTevXv10EMPKSioeHwQGRQUpEceeUR79+7VXXfdlaN9jx07prZt22rhwoX8oQJAEfafsO5jwroC4JprrlHfvn3/8ueBQEDx8fE6evTon75gGxQUpBdffDHfjs0wDHXu3JlBAgAAyKUiGdodPXpU119/vdasWZP9RpimRo8erbi4OHXt2rXYnhADBgzQ7t27NWTIkBztN3fuXHXu3DlX9wrKroDfr7RvFursU4/Jt2un43queg1U6sVXFdazj4wisuQZgKLLZRrq08jSWze51aSc81l3R84H9OQKnz6K9cmTmf8z4D6J8+uERxq/wS9/Ps/ymzRpknr16pWjpZdq1aqllStXavr06SpbtmyxPMeio6M1Y8YMLV++XDVr1sz2fh6PRz179tSUKVP4QwWAIoawruCaNGmSWrZsmaN9wsLCNGXKFF177bX5emyPPfZYjm/BAQAAgN8UudDu0KFDatu2rXbuzH6oU7NmTa1Zs0avvvqqwsLCiv1JUbp0aU2ZMkULFixQdHR0tvf7/vvv1aFDh3xZdsN3YL/OPfuEUj/9WPJ6HdUyQkMVPmioIp96TlbFSlwFABQqlUoYeuFGt+5vbSkiDz43W7rf1t8XefXjkfybAXfwjK2v9vxW//C5gL7Zm3/P9fbbb2vEiBGy7ew/x7Bhw7R161a1b9+eE0xShw4dtHXr1hx9gce2bd1999167733aCAAFAHFMaxzuVwFapZ9Vu/FS5QooR9//FFPPPGEIiIiLrqtaZoaMmSI9u7dqwEDBigqKipHxxIeHp6j7a+77jrNmTMnx0tvAwAAQDICl+NmZPlk//79ateunRISErK9T79+/TR+/HiVKFGCs+ECfv31Vw0YMEDLly/P9j6NGzfWypUrcxT4/ZVARrpSv/hc6UsWSXlwqrqvjFHEoKEyS0cxuAAKvfMZAU3Z5Nfa+LwJwWIqGRrR0qWy4Xn3zWg7ENDjy3zad/o/1/AQl/T+LW5FhebtN7DHjh2rJ598MtvblyxZUlOnTlXPnj05mf7CvHnzNHTo0BzNpH/55Zc1ZswYmgcABdxLPw9UUtof3ztHBkWpQ7X+urrirZclqBs1apTGjRuX5XYNGzbM0Rd1i7r09HQtWbJEsbGxOnHihJKSkhQeHq769eurfv36iomJUZ06dS75cfl8Pv3888/au3ev0tPTFR4errJly6p58+aqVClvvkAbHR2tkydPZrndc889p+eff56TBQAAFHhFZk3AI0eOqEOHDtkO7Nxut9544w098MADnAUXUa5cOX377bd69tln9dJLLyk7GW9cXJw6d+6sVatWOfpmXea2rfJMmyT7ZJLj38MoVVrhA4coOKY1gwqgyIgMNvTg1S61r2nrow0+nUhxVm/D0YDifvWqfxNLN9czZebBskaL9tp/COwkKd0nTd3s1yPX5N3LkHfeeSdHgV3Tpk01b968y/IBVmHSu3dvNW3aVL1799b27duztc/jjz+usLAwXmMBQGF6TXGZwzo4ExISoh49eqhHjx4F6rhcLpeuvfbafF+OEwAAoCgpEstjnjhxQh07dlR8fHy2ti9btqxWrVrFh0nZPUlMU//+97/1xRdfZHv50C1btujmm2+Wx+PJ8fPZyeeVPP4DJb8+Nk8Cu+B2N6rUy28Q2AEosppVMPV2V7d6NjRlOszZ0n3SlM1+jVnq08EzzmbwnfQE9PF2/wV/9sNhW9tO5M0MwUmTJumhhx7K9vZ9+vTRunXrCOyyqW7dulq3bp169eqV7X3+8Y9/aOrUqTQPAAo47lkHAAAAFCyFPrTzeDy66aabtHfv3mxtX79+fa1bt45veuVCjx49tHbtWlWsWDFb2//000/q3bu3fD5ftp8j48fvdXbMI8r84TvnJ3fFSop86jlFDB0hk3sVAijigl2G7mrm0utdXKob5XyG3P4zAY1e6tP0LT5l+HK3PPHEjT6lX+S/gIkbfPLZzpY+XrBgge655x5ld7XvMWPG6LPPPlNoaCgnTQ6EhYVp7ty5Gj16dLa2DwQCuvvuu/XVV1/RPAAooG6uOZSwDgAAAChgCnVoZ9u27rzzTm3evDlb27dp00Y//vijateuzcjnUsuWLbVu3TrVrVs3W9t/++232ZrR6E/6VedffUkpH72vQEqys4O0LIV266lS/35F7voNGTQAxUqNUqbGdnLp7isthThcfdIOSAt22/rHYq82HcvZrLgfj9iKPXrxIC0xWVq4O/ez7TZu3Kj+/fvLtrOuYRiGPvjgA7388ssyDIMTJRcMw9Crr76q9957L1s9tG1b/fr1y/brNADApdW8XDvCOgAAAKCAKdSh3UMPPaSvv/46W9u2a9dOy5cvV1RUFKPuULVq1fTdd9+pSZMm2dr+o48+0htvvHHBnwVsW2lLvtHZJx6VN26b42Nz1a6jkv96WWF97pDhdjNYAIrnf+6GoZvrWXrvZrdaV3YeUP3qkf69xqc3f/TpbHrWM9pSvQFN3pi9Wdaf7fAryZPz2XYJCQm67bbblJqamuW2lmVp2rRpGjVqFCdHHrj//vs1ZcoUWZaV5bYej0e33nqrEhMTaRwAAAAAAEAWCm1oN2PGDL377rvZ2rZTp05atGiRIiIiGPE8Ur58ea1evVrNmzfP1vZjxozRypUr//CYL/6Qzj3/lFI/nillZjo7oOBghQ0YqMhn/ilXlaoMEABIKhNm6PHr3XrsOpei8mA1yO8P23rgG6+W7/dfdDnKmVv9OpOevZqZfmnKJl+OjiMzM1O9e/fWsWPHstzWsizNmTNHAwcO5ITIQ4MHD9bs2bOzFdwdPXpUffr0kdfrpXEAAAAAAAAXUShDu23btmnkyJHZ2va6667T/PnzuXdNPoiKitLSpUvVoEGDLLf1+/268847lZiYqEBmplI/m6Nzzz0p/6GDjo/D3fxKlXr5DYV2uVmGaTIwAPA/rqpi6t2b3bqpjimn8+48XunDWL+eWelT4vk/B3d7Ttr6dl/Olrz8OTGQo+U3H3zwQa1fvz7L7QzD0JQpU3T77bdzEuSDO+64Q5MmTcrWUpnr1q3Tww8/TNMAAAAAAAAuotAlHMnJyerdu3e2lsO68sor9c033ygsLIyRzifR0dFavny5atSokeW2SUlJ6nPrLTr5+CNK+3qBZNuOntuILKmIkQ8o8uHHZJUpy2AAwEWEuQ0Nj3HppY4uVSvpfMnMnUkBPbTEq8/i/PL6fwvvfHZA42L9uao3aaPv9zoX8/HHH2vcuHHZqvn+++8zwy6fDR48WO+88062x+OTTz6haQAAAAAAAH+h0IV2Dz74oPbt25fldlWrVtXXX3+tyMhIRjmfVa5cWYsWLVKpUqWy3Hbdlq16fcUqx88ZfF1blXrlDQVffS0DAAA5UL+sqde7uDSgqaUgy1ktny19EufXw996tSvJ1vxdtg6fC+Sq1vEU6YtdF/8yx+HDh7N9X7rHHnuMe9hdIg888IAeffTRbG07cuRIHTlyhKYBAAAAAABcQKEK7RYsWKApU6ZkuV2JEiX0zTffqGLFiozwJdKwYUPNnTtXbrc7y23f2L5Tm06eyt0JW668Ih9/WhEjRskM5x6FAJAbLtNQ7yssvd3Vrablnc+6SzwvPbXCp0/i/I7qfLHTr+MpFw79bNvWoEGDdO7cuSzr9O7dWy+//DIDfQm98sor6tmzZ5bbnT17VoMHD77oPREBAAAAAACKq0IT2p06dUrDhw/PcjvDMDRz5kw1adKE0b3EOnTooDfffDPL7fyBgEb98LMy/Dn4cNc0FXLzbSr10mtyX9GYZgNAHqhQwtDz7d36extLJYKc17Md5jBeW5q80XfBn737ri2aywAAIABJREFU7rtavXp1ljWaNGmiGTNmZOs+a8jDF5SmqVmzZqlx46z/j165cqXee+89mgYAAAAAAPA/Ck1o9+ijjyopKSnL7R577DF1796dkb1M7r//fvXr1y/L7fadT9Zb23dmq6ZVo6ZKvvCSwu8cICMoiCYDQB5rV9PS+7e41a7G5X9ZsPFYQOsT/rhM5pEjR/TMM89kuW9kZKS++OIL7mV7mYSFhWnevHnZWpr86aefVkJCAk0DAAAAAAD4L4UitFu9erWmTZuW5XZt27bViy++yKheZhMnTlSDBg2y3O6dHbu199z5v94gKEhhd/RXyedflKt6DRoLAPmoRLChv1/l0vPtXKpwmVcfnrzJpwzff6bt3X///UpJSclyv6lTp6pOnToM5mVUr149TZo0KcvtkpOT9cADD9AwAAAAAACA/1LgQzufz6dRo0ZluV1kZKRmzJghy7IY1cssPDxcs2bNyvL+dl7b1uPrN17wZ+7GTVVq7OsKvaWbDNOkqQBwiTStYOrtm9zqfYUp6zKtMJmUKs3d+dsSyl999ZUWLlyY5T5DhgxRr169GMAC4Pbbb9egQYOy3G7+/PlatGgRDQMAAAAAAPj/CnwaMmHCBO3atSvL7d5//31Vr16dES0gWrZsqWeffTbL7dYe/1XfJhz9/d9GRITCh49U5GNPyoouRyMB4DIIsgwNaOrS611cqlfm8iR3C3bbOnzaq9GjR2e5bc2aNfXOO+8wcAXIu+++m63XZY8++qj8ObnHLQAAAAAAQBFWoEO78+fP6/nnn89yu5tvvll33XUXo1nAPP7442revHmW2z2/cYt8tq2gq65RqVfeVMj1N9A8ACgAqpcy9VJHl4a3tBTqurTP7bOlkS+M0549e7LcduLEif+PvfsOj6Ls+jj+280mgSSUBEIIhITeVVCagKBUBSmC5QEBBREREOwiKiBWQJAmCKg0FRFQAUWlNxUIICV0Qgg1PZDe8/7xPPISk+xuICGb3e/nurwuM3Nmdvac2XBnz8w9KlOmDAWzIWXLltWCBQssxh0/flwLFy4kYQAAAAAAALLxpt3UqVMVGRlpNsbNzU2fffYZlbRBJpNJ8+fPl9HC9Jan4+L1Q636KjNitIxlypI4ALClgYLBoIfqOGl2d2e19Lt9d91lpCRq06L3LMYNGDBAHTt2pFA2qEuXLurfv7/FuIkTJyopKYmEAQAAAAAAh2ezTbvY2FjNmjXLYtz48eNVvXp1KmmjWrRooeHDh1uMm7J0mdLT00kYANgor9IGvdHWWWPbmlTOtehfL2TTfKXFR5mNKV++vKZPn05xbNj06dNVrlw5szHh4eH6/PPPSRYAAAAAAHB4Ntu0mzFjhuLi4szGVK9eXS+++CJVtHHvvvuuxS/szp07p6VLl5IsALBxzaoa5O1etHfcZaQm6dQ6y824t956S97e3hTFhvn4+GjcuHEW46ZOnark5GQSBgAAAAAAHJpNNu0SEhKsusvu448/lqurK1W0cRUrVrTqC7uPP/5YWVlZJAwAbNhvZ7J0Jia7SF8jdMtXSoszPz12jRo19MILL1CQEmDMmDEKCAgwGxMWFqYvv/ySZAEAAAAAAIdmk027xYsX6+rVq2Zj7r77bj3xxBNUsIQYM2aMqlSpYjbmzJkz+vnnn0kWANiomORsfXMos0hfIzsrS8G/WX5W7aRJk7hwp4RwdXXVu+++azFu1qxZys7OJmEAAAAAAMBh2VzTLjs7W7Nnz7YY984771C9EsTV1VVvvPGGxbiZM2eSLACwEQlhOZ81unB/hpIzivY1w/5er8SIs2Zj6tSpo379+lGgEmTAgAGqVauW2ZjTp09r/fr1JAsAAAAAADgsm2va/f777zp16pTZmLvuuku9evWieiXMsGHDVLlyZbMxW7Zs0bFjx0gWANiAoO9jtWtquCKOJWvvxSztuVj0d0EF/z7XYsxbb70lJycnClSCODk5WTVV9pw5c0gWAAAAAABwWDbXtPvqq68sxrzyyisyGAxUr4QpVaqURo0aZTFu0aJFJAsAbMTVc2na+1mUTsyNUNXY1CJ9raSo84oM2mI2pkqVKurfvz+FKYEGDBhg8eKdDRs26OLFiyQLAAAAAAA4JJMtHUxsbKzWrl1rNqZy5co8y64Ee+655/T+++8rJSUl35hly5bpo48+kslkImEAYCO8rqWr67WriijjrL8D3HXJs/CfJ3d+x9eShWeaPf/883J2dqYgJZCLi4uef/55TZgwId+YrKwsLV261Kq78gAA9ufMmTNWxYWFhWnkyJEkDEpISLAqztKMTgAAALbCkJ2dnW0rBzN37lyLA+8JEyZo4sSJVK4EGzJkiMW76X7++Wd1796dZAFAMdo9K0JRJ/O+u64omncbXmyoxPDgfNe7uLjo4sWL8vb2pjglVEREhPz8/JSenp5vTN26dXXy5EmSBQAOqHHjxjp69CiJQKFr3769tm3bRiIAAIDNs6npMVetWmV2vcFg0ODBg6laCffMM89YjFm5ciWJAoBiZu6qnkrx6eoadFUPH4wplGkzr4YcNNuwk6SePXvSsCvhKlWqpB49epiNOXXqlA4fPkyyAAAAAACAw7GZpl1UVJR27NhhNqZDhw4KCAigaiVcmzZtVKdOHbMxa9euVUZGBskCgGIUmWj5ZvzCat5dDvzJYgwX7tiHp59+2mLM6tWrSRQAAAAAAHA4NtO0W7NmjTIzM83GDBw4kIrZCUu1jI2N1datW0kUABST8IRshSdaH3+rzbvLe8037SpVqqSuXbtSGDvw0EMPqWLFimZjfvjhBxIFAAAAAAAcjs007X799Vez652dndWzZ08qZif69u17y+cEAKDofL4vQ1k38dTbm2neJUWdV/yl42ZjevfuLScnJwpjB0wmk3r16mU2JigoSBcvXiRZAAAAAADAodhE0y4rK0tbtmwxG/PAAw/I09OTitmJhg0bqn79+mZjNm7cSKIAoBjsOJepQ2HZt7SPgjTvIo5stri/Pn36UBg7Ys3FO4wDAAAAAACAo7GJpt2+ffsUGxtrNoa77OxPjx49zK4PCgrSlStXSBQA3Gbnr2Xrbl+DSptufV/WNO8iDptvznh4eKhDhw4Uxo506NBBbm5uZmNo2gEAAAAAAEdjE0277du3W4zp3Lkz1bIzXbp0sRizY8cOEgUAt9mAu0x6u72z6lYwFNo+zTXvoo7vMrtt+/bt5ezsTGHsiKurq9q1a3fL40MAAAAAAAB7YhNNu927d5td7+/vr7p161ItO9O2bVuVKlXqls4NAEDJ8u/mXVJkqFKvhZvdhgt37JOlul6+fJnn2gEAAAAAAIdSIpp2lq7ERslUqlQptWjR4pbODQBAyfRP8+7uP7ZYjGUcYJ/at29/y2NEAAAAAAAAe2Iq7gO4cuWKLl++bDamVatWVMpOtWrVyuwUmAcPHlRmZqacnJxIFgDYoRMnD5hd7+bmpjvvvJNE2aG77rpLpUuXVnJycr4xgYGBevTRR0kWADiIgIAAHT161GJcxYoVNXr0aBIGffzxx0pKSrIYV716dZIFAABKhGJv2gUFBVmMoWlnvyzVNiUlRWfOnFG9evVIFgDYodDok2bXN2vWjAs37HUQajLpnnvu0a5du25pnAgAsB8BAQFWxXl7e+udd94hYdCsWbNo2gEAALtS7E07S1fROTk5qXHjxlTKTjVp0sRizI41++XRwY9kAcBtlhqXWeSvcT721C3/O4GSPQ4w17Sz5m4LAAAAAAAAe1HsTbtjx46ZXV+7dm25urpSKTtVvXp1ubu7KzExMd+Y7av/lm/ofSQLAOxMUlqCohKumI1p1KgRibJjlup7/vx5JSYmyt3dnWQBAAAAAAC7ZyzuAwgJCTG7ni/r7JvBYFCDBg3MxoTHnSdRAGCHIuIvWoxhHGDfLNU3OztboaGhJAoAAAAAADiEYm/aWfoipmbNmlTJzlmqcWTCZZIEAHbImqYd4wDHHgNYM1YEAAAAAACwF8XetLtw4YLZ9dY+iBoll6UaW/OlLgCg5ImMN39RhqurqypXrkyi7Jivr6+cnZ3NxtC0AwAAAAAAjqJYm3bXrl1TSkqK2Rh/f3+qZOcs1fhqUhRJAgA7dDU50ux6Pz8/GQwGEmXPA1GjUVWrVjUbEx4eTqIAAAAAAIBDKNamXUxMjMUYb29vqmTnLNU4OT1BmVkZJAoA7Ex8ylXGAFClSpVuebwIAAAAAABgD4q1aRcdHW0xxsvLiyrZOWtqnJB6jUQBgJ2JT4llDACLdaZpBwAAAAAAHEWxNu3i4uIsxnh6elIlO2dNjRPT4kkUANiZpPQEs+tp2jEOsHa8CAAAAAAAYA+KtWmXlpZmMaZ06dJUyc6VKlXKYkxGZhqJAgA7k5GZfsv/PsD+xwGpqakkCQAAAAAAOASbb9q5uLhQJTtnTY0zstJJFADYmYysNMYAsFhna8aLAAAAAAAA9sBUnC+enm65EWMymaiSnXN2drYY03ykp+65uwrJAoDbaN/CKMWcKbqGSUZWBmMAWBwHZGRkkCQAAAAAAOAQivXbMGuaNRkZGXJycqJSdsya5m3psq5y8eA8AIDbyehkKNpBiNFkcQwAxgE0bwEAAAAAgKMo1ukxrZn2iimR7B/TpAKAYzIZmRYRluvMGAAAAAAAADgKm2/aJScnUyU7l5KSYjHG1dWVRAGAnbF0xz1jAMYBjAEAAAAAAIAjKdamXbly5SzGxMbGUiU7Z02Ny5YtS6IAwE44uRpUu2sZ1WjmzRgAFuvMGAAAAAAAADiKYn1IiJeXl8WYmJgYqmTnrKmxp6cniQKAEs7J1aAa93uoZscycnF3UqWN3owBoOjo6FseLwIAAAAAANgDm2/aRUZGUiU7FxERYXZ92bJlZTKZSBQAlFD/btZZOw5gDOAYLNW5QoUKJAkAAAAAADiEYu2ElCtXTqVKlTL7LJPz589TJTt34cIFs+t9fHxIEgCUQPk166z9/X7x4kVlZ2fLYDCQTDuVlZWlS5cumY2pVKkSiQIAAAAAAA6h2G9f8vf316lTp/JdHxoaSpXsnKUaBwQEkCQAKEEsNeus/f2empqqsLAw+fr6klQ7deXKFaWnpzMOAAAAAAAAUAlo2p09e5Yq2TlLNebLOgAoGaxt1t04BrDm3wiadvYrODjYYgzjAAAAAAAA4CiKvWlXo0YNs+uPHj1KlexYdna2jh07ZjamevXqJAoAbFhBm3UF+f1+9OhRtWnThiTbKUtjAIPBQNMOAAAAAAA4jGJv2jVs2NDs+jNnzig1NVWurq5Uyw6dO3dOSUlJt3SOAACKx8026/5RpkwZVatWzeyzTbl4x74FBQWZXe/v7y93d3cSBQAAAAAAHEKxN+0aNWpkdn1mZqaCgoJ0zz33UC07dPDgwVs+RwAAt9etNuv+/TveXNPOmn8nUHIdOnSIMQAAAAAAAMD/GIv7ABo3bmwxZvfu3VTKTlmqbalSpVS7dm0SBQA2wMnVoNpdy6jje76q37P8LTfsrBkH7Nu3TxkZGSTfDqWnp2v//v23PE4EAAAAAACwF8V+p52vr6+qVKmiy5cv5xuze/dujRw5kmrZIUtNu6ZNm8rJyYlEAUAxKsw76/6tWbNmZtcnJSXpyJEjatq0KYWwM4cPH1ZycrLZmObNm5MoAACK2bJly3Tx4sUcy1577TWZTCaSAwAAUMhsYoTVqlUr/fDDD/mu37FjB5WyQykpKdq7d6/FcwMAUEyDhFJG1e5apkiadQX5Pb9jxw6adnZo+/bthXJ+AACAojVq1CjFxcXlWPbSSy/RtAMAACgCRls4CEtfyJw/f16nTp2iWnZm586dSklJMRvTsmVLEgUAxeTuZyoU2jSY+QkICFDlypXNxmzcuJFi2CFLda1atar8/PxIFAAAJUhWVpbi4uJy/JeQkEBiAAAArGQTTbv27dtbjOELO/tjTU3btWtHogCguAYJTgabGAds375d6enpFMSOpKamWpxJgTEAAAAlz5EjR1SuXLkc/zHdNQAAgPVsomnXrFkzeXp6mo1Zu3Yt1bIz69atM7u+cePG8vX1JVEAYOc6d+5sdn1CQoK2bNlCouzI5s2blZSUdEvnBQDg1mwKzlRiWjaJAAAAAGyITTTtjEajOnToYDZm69atio2NpWJ24tixYzpx4oTZGL6sAwDHYM3v+9WrV5MoO2LuWcaMAwDg9vjxRKaeW5eu74No3gEAAAC2wmgrB/LQQw+ZXZ+ens7ddnbEmi9fLZ0TAAD74O/vr4YNG5qNWbNmjTIzM0mWHcjIyNCaNWvMxjRu3Jjn2QHAbZCULn0XRPMOAAAAsBU207Tr1auXnJyczMYsXbqUitmJZcuWmV3v6empBx54gEQBgIPo06eP2fURERH6/fffSZQd+PXXXxUVFWU2pm/fviQKAG4jmncAAACAbbCZpl3FihXVrl07szFbt25VaGgoVSvh/vjjD50+fdpsTM+ePWUymUgWADgIa5o0ixYtIlF2YPHixRZjLDVxAQBFg+YdAAAAULyMtnQwjz32mNn12dnZfGFnB7788stbPhcAAPalSZMmql27ttmYtWvXKjIykmSVYBEREVq3bp3ZmLp16+rOO+8kWQBQjGje5S87O1sXLlzQli1btHr1au3Zs8fq8Ul8fLwOHz6s9evXa/Xq1dq5c6fOnDmjrKwsEgsAAABJkk3dyvSf//xHL730klJTU/ONmT9/vsaNGycXFxeqVwJFRUVp+fLlZmN8fHzUtWtXkgUADuapp57SO++8k+/6tLQ0zZ8/X2+//TbJKqE+//xzpaenWzwPAAC24Z/m3dqTmepZz0nd6xrl7mKwy/eakJCghx9+OMeyLl26aNy4cZKk6OhozZgxQ3PmzNHVq1dzbe/n56e3335bQ4YMkbOzc45169ev1/z58/Xrr7/m+e+gn5+f+vXrpzfeeEMVKlQwe5yhoaG5/q2sV6+e5s+fb/V7Xbx4cY47341Go6ZNm6amTZveVO5ee+01BQYGXs/jv50/f173339/jmWbN2+2+IgUAAAAR2RTTTtPT0/17NlTK1euzDcmLCxMK1as0MCBA6leCTR//nylpKSYjRk4cCBTYwKAAxo0aJAmTJhg9mrzefPm6Y033sj1ZRhsX1pamubNm2c2xmg0atCgQSQLAGyMIzTvMjIytH379hzLLl68qHHjxmnVqlUaPHhwng2pG2OHDx+uWbNmacuWLfLx8dG1a9c0cuRIffPNN2Zf++LFi5o6daqWL1+uVatWqWXLlvnXIikp13EeP368QE27kJCQXPv47bffbrppd+jQoVz7s3TM2dncvQkAAJAXo60d0JAhQyzGTJs2jQFeCZSSkqI5c+ZYjBs8eDDJAgAH5O/vr44dO5qNuXz5sr799luSVQJ9/fXXCgsLMxvTpUsX+fn5kSwAsFGOOG3m/Pnz9fjjj5tt2N3o2LFj6tGjh+Li4tSpUyeLDbsbXbx4UT169FB4eDgnGwAAgIOyuaZd165dVbduXbMxhw4d0po1a6heCbNgwQKLX9Z16NBBDRs2JFkA4KBeeOEFizEffPCBMjMzSVYJkpmZqQ8//LBQ6g8AKH6O0rwLDg7WiBEjCnzRcGBgoMqXL699+/YV+DUjIyP14osvlqg8Mc0lAABA4bG5OQgNBoNeeOEFi1/avPfee+rduzcVLCFSU1M1efJki3El7Y8TAEDh6t69u2rVqqXg4OB8Y06fPq3ly5drwIABJKyE+Prrr83WVJLq1Kmjhx56iGQBQAniCNNm3jhtt7u7u/r166dmzZqpXLlyOnLkiL777judPXs213b/bvQ5Ozvr0UcfVdOmTeXn56eLFy9q8eLFOnbsWK5tf/rpJyUkJMjDw6NE5GjmzJnasGGDpP9eZP3FF1/kWO/j45PrucU0+gAAAPJmkw8Oe/rpp/XOO+/k+XDnfxw4cEArVqzQE088QRVLyCD+8uXLZmNq166t7t27kywAcGBGo1GjR4/WmDFjzMaNHz9ejz32mFxdXUmajUtNTdWECRMsxo0ZM0YGg4GEAUAJ5AjNu3bt2mnRokWqWbPm9WX/+c9/NHr0aHXq1ElBQUH5btuyZUstXLhQd9xxR47lr7zyil588UXNnj07x/KUlBT9+uuveuyxx0pEburWrXt9xqS8mnaenp4aOXIkHxQAAAArGG3xoDw8PDR69GiLcWPHjlVqaipVtHFRUVFWTYn15ptvymg0kjAAcHBDhw5VpUqVzMaEhITk+oILtmnmzJkKDQ01G1O5cmWrnmsMALBt9jpt5gMPPKBt27blaNj9w8fHRwsXLsx32zp16mj79u25GnbSfy9W+uSTT1S7du1c644cOcIJBQAA4IBstkPy4osvqmzZsmZjzp07pxkzZlBFGzd+/Hhdu3bNbEz16tU1cOBAkgUAkJubm1577TWLcR988IEiIyNJmA0LDw+36sKd1157TaVLlyZhAGAn/mneDbeD5p3RaNT06dPN3g3eokULeXl55blu9uzZZmcGcHFxUevWrXMtZ4wDAADgmGy2aefp6WnV3XaTJk3SuXPnqKSN2rt3r+bPn28xbty4cXJ2diZhAABJ0vPPPy9vb2+zMVevXtXLL79MsmzYSy+9ZPHCHR8fHw0fPpxkAYAdSvxX8y4pveQ17wYNGqQmTZqYjTEajapatWqu5S1atFDXrl0tvsY/U0veiKYdAOSUnJysX3/flOM/ALBHNj0X4WuvvWbxC7ukpCTmRrdRGRkZeu6553I8uDsvDRo00ODBg0kYAOA6d3d3q56D9vXXX2vTJv5Ys0UbNmzQ8uXLLcZNnDhRbm5uJAwA7Ng/zbvn1pa85t2DDz5oVZynp2euZfXr17dqWw8Pj1zLkpKSOHEA4Aa79+7Tl4u/zvEfANgjky0fXNmyZTVx4kSLTbn169dr6dKlGjRoEBW1IR9//LEOHjxoMW7KlCkymUwkDACQw3PPPafZs2fr5MmTZuOGDRumQ4cOqUyZMiTNRsTFxWnYsGEW4xo0aKBnn32WhAGwS4lp2Vr0d6bNHt/V5GLIyf+ad+tOZqpHPSc9XM8oN2eDTdfR39/fqrhSpUrlWpbXM/AAADdn5x+7SQIAh2DznZJhw4Zpzpw5On78uNm4F154Qe3bt1dAQABVtQH79+/XpEmTLMZ17NhRDz/8MAkDAOQepJhMmjp1qnr27Gk2LiQkRGPGjNFXX31F0mzE6NGjFRoaajHuk08+kZOTEwkDYJdSMqQtIVkkIg8lqXl3K98x0LQDgMIRe/WqgoKOkQgADsFo6wdoMpk0d+5ci3FxcXEaNGiQMjMzqWpx/wGWmKgBAwYoPT3dbJyLi4s+++wzEgYAyFePHj0sNu0kadGiRVq9ejUJswErV67UkiVLLMY98sgj6tatGwkDAEf+2/Ff02amZ9rWtJnOzs6qXLnyTW/P9M8AcOvSMzI0c858ZWVnkwwADsFYEg7y/vvv19NPP20xbseOHXrrrbeoajF79tlndeLECYtxY8eOVb169UgYAMCsOXPm5Pmsl38bMmSITp8+TcKK0alTpzR06FCLcWXKlNGsWbNIGABA0v8377afs607Ez09PWU0Gu0u32lpaZx0AEqE+PgEzf5svoKOHicZABxGiRl9fvLJJ/L29rYYN2XKFK1Zs4bKFpM5c+Zo+fLlFuPq1auncePGkTAAgEXVqlXTe++9ZzEuLi5Offv2VVJSEkkrBklJSerbt6/i4uIsxn7wwQfy8/MjaQCAHDKYTfS2iI2NJQkAbFJqaqpCzoVq+84/9PEnMzT0+TH6c3cgiQHgUEwl5UArVKighQsXqnfv3mbjsrOzNXDgQP3xxx+64447qPBttHnzZr388suWTzqTScuWLZOrqytJAwBYZfTo0VqzZo22bdtmNu7IkSMaNGiQVq5cKYPBQOJuk6ysLA0YMEBBQUEWYzt06KBRo0aRNAAAiglNOwC2ZvPWHfp+1U+KjokhGQAcnqkkHWyvXr00ZMgQffXVV2bj4uPj1b17d+3Zs0e+vr5U+TY4duyYHn30UYvPsZOkd955R82bNydpAACrGY1GLVmyRHfeeaeuXbtmNnb16tUaO3asJk+eTOJuk9dff10//vijxbjy5ctr8eLFNFQBAChG4eHhJMGOZWZl6dKlK7p85YqcTSbVrl1T5cqWtXr7qOhoxcZeU3x8vFxdXeXq6iLvihVVrlzZEpuTq1evKSw8XFfCIpSSkiJ3dzeVLVNGPj6V5FvZp1Bfyx7zdztERUfTsAOA/zGVtAOeOXOmduzYoTNnzpiNu3Dhgh5++GFt3bpVZcvyD2NRunTpkrp3766rV69ajG3dujXPHQQA3BR/f3/NnTtXTz75pMXYKVOmKCAgQCNGjCBxRWz27NmaNm2aVbGff/65qlWrRtIAAChEmZmZVsempaUpMJCp5kqqDyZPV2pq6vWfm99zt3p07ypJSk5O1tJvVmj7jj+UdsMF1S+MeFbt72tjdr/nL1zS5q3btTfwgCKjovKMqVixgtrf11r339dGvr6VLR5rZGSUZs9bmGNZFV9fDX/2aavf79btO7V1+67rPxsMBj01oJ9q1giwuG14eIS+XbFa+/8+pJSUlHzjKnlXVLN7muqRnt3l6Vn+pupSFPkDADiuEte08/Dw0OrVq3XvvfdafGbNgQMH1K1bN23YsEFubm5UuwhERkaqU6dOOnfunMVYb29vff/993JyciJxAICb0r9/f+3atUvz5s2zGDtq1Ch5eHho0KBBJK6ILF68WGPGjLEqdtSoUXriiSdIGgAgT82rGuTmb1QIqTArr7vVo6OjFRUVpYoVK1rcfvfu3Tz/twQ7cfKUkpP/vwGVmJikHt276uSpM5o+c26B71RKTEzU8u9/0IaNW5SVnW02NioqWqt/XKfVP67Tg1066sl+j6l0qVL5xqd/7+ftAAAgAElEQVSmpenY8ZM5ll28dLlATbvwiKhc+zh46IjZpl1iYqK+X71Gv2/YrAwrGtoRkVFa/9tGbd66Xf0e76uHu3W1ifw5GhdnZ5IAAP9jKokHfeedd+rzzz+36ku4P/74Q7169dLatWtVunRpKl6IYmJi1KVLF504ccJirJOTk7777jtVrVqVxAEAbsmMGTO0f/9+7d2712xcdna2hgwZotKlS+uxxx4jcYVsxYoVGjp0qLItfEEhSa1atdL06dNJGgAgl+ZVDfpPYyfV8DRqxHKmT7akQoUKeS4/cOCAunTpYnH7JUuWkEQ7E3IuVB98PE1JyckF2i4mJlaTPpyqi5cuF/g1f9uwWfsOHNTYV8eoeoC/zeQiIyNDH02doRMnTxd429TUNC1etlzJycl6rG9vh8xfcerSuYMaNqif7/ozwWe1aOm3JAqAQzCV1AMfOHCg9u3bp1mzZlmM3bRpk7p166Z169bJw8ODqheC8PBwderUSUFBQVbFT5kyRR06dCBxAIBb5uLiotWrV6tFixa6cuWK2djMzEz169dPycnJ3HFXiBYvXqyhQ4daNR1XlSpVtGrVKjlz9SwAR/tj2yhVL2+7TaiLcdnKyCq+17+xWQfrVahQQSaTSRkZGTmW79+/32LT7uDBg1q8eDFJtCPxCQl6/yYadtExsXpn4geKiIy66deOiorW+x9P04eT3lYlb2+byMdXS74x27AzGgwW74hbseon1aldS03uusPh8lec3N3cVK9u7XzXm5viFADs7u+Iknzwn376qc6ePauff/7ZYuy2bdvUqVMnrV+/Xl5eXlT+Fpw/f16dOnXS6dPWXbk0fPhwvfzyyyQOAFBo/Pz8tG7dOrVv316JiYlmYzMzM/X0008rISGBZ9wVgjlz5mj06NFW3WHn7u6un3/+mTvtATikcqUMmv6g7V6wMPKXNF2Jv/2vS7Pu1hiNRvn6+urChQs5lk+ZMkUPPfSQmjRpkud2p06dUq9evZSVVbSd2rym77x06ZKysrJkNFLzwhYTE5vvuvymG8zKytKsz+bn2XAyGAxq1/Ze1a9XV9WqVVVSUrJOnQ7W6dNndOzEKaXf8Kw8Sbp69Zo+mjJDn0x+T07FXN+zIaHasGlrruX+1fzUq0c33XVHI5UrV1Zp6ekKD4/U5StX9OOanxV89lyubb5evlJ33dk4z/PZXvMHALAdJbppZzQa9d133+m+++7T33//bTF+z549at26tX755RfVqlWL6t+E/fv3q0ePHhbvbPhH165dNXv2bBIHACh099xzj7755hv16dPH4hdQ2dnZGjlypM6fP6+PPvoozz/AIYs5fOONNzR16lSr4v+ZGrtp06YkDwBAs64Q9ejRQ3Pnzs2x7OrVq+rSpYu++OILtW3b9vrFytHR0VqwYIGmTZum6OjoIj+2SpUq5VoWHx+vN998U8OHD1dAQIBSUlLk5uZGIYtA/Xp19PBDXRTgX00+PpXybJT+8usGHT2W+zEnFby8NHrkMDVqmHOKwrub3ClJOn7ilD6YPD3XHU8XLl7Sjp1/6IH29xXre9//98Fcy2rVrKFJ48fK1dX1+jJXFxf5V6sq/2pV1bzZ3fpl/e9a+s2KHNudCz2vo8dPqHHDBg6TPwCA7TCV9Dfg7u6uX3/9Ve3atdOpU6csxp88eVKtWrXSTz/9pDZt2nAGFMBPP/2kJ5980uqHVt97771avXq1TCYTyQMAFIlevXppwYIFevbZZ62682vy5MkKDg7W0qVLedZtASQlJWngwIH64YcfrIo3GAxauHChHn74YZIHAA6OZl3he/LJJ3M17SQpMjJSvXr1ksFgkL+/v9LT03XlyhWrxkiFxcfHR66urkpNTc2xfMqUKZoyZcr1n9PT0/muoBAZDAY90uthPfHYI2bv2EpNS9MPa3LPVuVdsaKmfvSuPDzc8922Qf26evvNV/TBx9OUnJyz8bT6x3XF37Q7kLtp1/eRHjkadv/mZDSq58MP6fjJ0wrcdyDHuvPnL+Zq2pWE/KWnp2vWZwtuS85fGv08d9ACQBGwixGSj4+PNm3apPvuu0+hoaEW46OiovTAAw9o2rRpeuGFFzgLLMjKytL48eP14YcfWj3Yb9KkidavXy93d3cSCAAoUs8884wSEhL04osvWhW/atUqnTp1SqtXr1bt2rVJoAWnT59W3759deTIEau3mTlzpgYPHkzyAMCB0awrOq1bt9bjjz+u77//Ps/12dnZ+X434uLiogkTJuitt94qkmMzGAzq0qWL1q1bR6Fuox7dH1T/J/pajNux80/FxyfkWv7c0KfMNpz+Ub9uHY0Y9oymzfwsx/Kw8AhdunxFVav4FlsOwsIj8vowWLXtA+3b5mraXbp8pUTmLzMzS3/tCbwtOX9Rz/PhA4AiYDej52rVqmnz5s3y8/OzKj49PV2jR49W//79FR8fz5mQj4iICHXt2lUffPCB1Q27xo0ba8OGDSpfvjwJBADcFmPGjNGHH35odfzhw4fVrFkz/fjjjyTPjNWrV6tZs2YFathNnjyZi6IAwIE1r2rQtK4mvXmfMw27IvTFF1/onnvuKdA2bm5u+uqrr4p81qHXX3+dqchvo/LlyumxPj2tit2+849cy1q3aq4md91h9eu1aHGPvDxzf99z+MjRYs2Dk5NTrmUbNm9TekaGxW0bN2yg0SOH5fiv/X2tHSp/AADbYVcj6Fq1amnnzp0Fel7d8uXLddddd2nXrl2cDf+yZs0aNW7cWJs2bbJ6m2bNmmnbtm3y9vYmgQCA2+rNN9/Up59+avWXRNeuXVOfPn00dOhQJSQkkMAbxMfHa8iQIXr00UcVFxdn1TYGg0GzZs3S66+/TgIBwAE1r2rQJyW4WWcymeTi4mIzx2Ppb+oyZcrozz//1JtvvikPDw+zsUajUYMHD9apU6f05JNPXn/enbUKOoNO27ZttXz5ci7kvU26P9TZqmnf09PTdSY4JNfyli2aFej1nIzGPLcJORdarHmoVzf3DBoHDx3RR1M+1ZWwcLPburmVVru2rXP8V7dObYfKHwDAhsal9vaGqlevrh07dqhz5846duyYVduEhISoffv2euWVVzRx4kSHfyBybGysXnnlFS1atKjAA/NffvlFZcuW5ZMFACgWL774ojw8PPTcc88pKyvLqm2+/PJLbd26VQsXLlSHDh0cPoebN2/Ws88+q5CQEKu3MRqNWrhwoYYMGcJJCAAOpnlVg55o7KSaJfyuOg8Pj1zPYSuI33///aa3HTNmjMaMGVPg7VxcXPThhx9q/Pjx+u233xQYGKjw8HBFRkbK3d1d9erVU7169dSsWbMcU4LfcccdBXrO3eXLlwt8bE888YT69u2rPXv26NSpU0pJSZG7u7sqVqyoJk2a8Dy7QlSjeoBVcWeCQ5SRx11nNa3c/kYB/tVyLYuIjCrWPDRqUF97Aw/kWn74yFGNeXmsGjasr3uaNlHjRvUV4F+twM9iKyn5M5mcVKmStyIiIvlwAEAJZZejpCpVqmjnzp3q06ePtm/fbtU2WVlZmjp1qlatWqW5c+fqwQcfdMgT4ptvvtHLL7+siIiIAm336KOPaunSpVZd3QUAQFEaOnSovL291b9/fyUlJVm1zdmzZ9WxY0cNGjRI06ZNU8WKFR0ub5GRkXrllVe0bNmyAm3n7u6u5cuXq0ePHpx8AOBA7KVZZw9KlSql3r17q3fv3jZ1XCaTSW3atCny6TgdXTW/qlbFRUTm3cSZO/8rqYCzmV68lLuRGxN7tVjz0LpVC61b/7uioqJzrcvKzlbQ0eMKOnpc0n+nim1Qv67ubNxQd97RyKoclpT8mUwmzfl0stUXMN4KJyO//wGgSMZQ9vrGvLy8tGHDBg0dOrRAXz6FhITooYceUs+ePTVlyhTVq1fPIU6EwMBAvfLKK9q5c2eBt3399df18ccfM2c9AMBm9OrVS9u3b1ePHj0UFhZm9XZLly7VunXr9M4772jkyJE2NU1WUUlLS9Ps2bP1/vvv6+rVgn1ZUKVKFa1bt0533303Jx0AOAiadYBtKVPGw6q4hITEPJcfP3mq0MaUxcnTs7wmjR+r8ZM+zrNxd6OkpCTtP3BQ+w8clCRVrFhBbe5tqfb3tZF/taolPn9Go7HAdxICAGyHXf8Gd3Fx0dKlS/XRRx/l+UBac9auXavGjRtr5MiRunTpkt3m6PTp03ryySfVsmXLAjfsSpUqpUWLFmny5Mk07AAANqdZs2bau3evmjdvXqDtYmNj9fLLL6thw4b69ttvb8tVqsUhMzNT33zzjRo0aKBXX321wA27li1bas+ePTTsAMBB3PjMOhp2QMkTn0/TqbBkpGcU/j4zCrbPSt7eem/8m2rZolmBvqeKiorWmnXr9fLrb2nqp3OUmJhoF/kDAJRMDjHSHjt2rDZs2KBKlSoVeHAwd+5c1apVSyNHjtT58+ftJifHjx/XgAED1KBBA3377bcFms9ekmrWrKm//vpLTz/9NJ8iAIDNqlatmnbt2qXhw4cXeNvg4GA9+eSTatiwob7++usCf2lgqzIyMrR06VI1bNhQAwYM0NmzZwu8jxEjRmjHjh3y8/PjJAMAO0ezDrAPWZmZRbp/g7HwL+bOq3lmibd3Rb320ijNmv6xHu7WRT6VvAu0/Z69+zR52iylp6eX+PwBAEomh3nyb4cOHXTgwAENGDBA27ZtK9C2qampmjt3rubPn68+ffpozJgxJXJO9uzsbP3++++aOXOmfv/99wI36v7Rt29fffHFFypfvjyfIACAzXNxcdG8efPUpk0bjRw5UnFxcQXa/uTJkxo4cKDefPNNjRgxQsOGDVOFChVKXB6io6M1f/58zZ0796ZnEShXrpzmzp2r/v37c2IBgJ1jGkzAvrh7uOe5/LWXXiiU/VepUrnQj/lmmnb/8K3so6cH9tfTA/vr8pUwHT5yVEHHjuvYsZOKi483u+2x4yc1b8FXGj3yuRKdPwBAyWRypDdbtWpVbd68WdOnT9fbb7+t1NTUAm2fmZmplStXauXKlWrcuLEGDx6sAQMGFPgOvtstNDRUS5Ys0ZIlS27qavp/lC1bVrNmzdJTTz3FJwcAUOIMGDBA9913nwYNGqQdO3YUePuLFy9q3LhxmjRpknr16qXBgwerc+fONv28iMzMTG3cuFGLFi3SmjVrCjz2udH999+vJUuWyN/fn5MJAOwYzTrAPnm45246GQwGNW1yh80+x/nqtbhC2U8V38qq4ltZD3bpqOzsbIWev6iDhw7rwMHDOnb8ZJ7b7Ppzj4Y/O/h6bkpK/rKysnT+wu15zE/1gGp8sACgCJgc7Q0bjUa9+uqr6tq1q5555hkFBgbe1H6CgoL0yiuv6I033tD999+vvn37qnfv3qpc2TaujDl37px++OEHrV69Wn/99ddN31X3jy5dumjBggUKCAjgUwMAKLECAgK0detWzZgxQ+PHj7+pq3dTUlK0YsUKrVixQpUrV9Yjjzyivn37qn379jKZin9olZGRoW3btmn16tX66aefFBYWdkv7c3d313vvvacxY8bwQHsAsGM06wD75umZe7ak7OxshYVHyL/a7ZnyvCDPik7PyNCZ4JBCPwaDwaDqAdVUPaCaevfsrith4fp8wSIdPX4i17GeCz2vunVq20z+rJGWlq5Xx75zW15rxTdfyYm/DwCg0Jkc9Y3fcccd2r17t+bOnau33nqrwFNl/SMjI0ObNm3Spk2bNGLECN11113q3LmzOnbsqJYtW962KSQjIyP1559/avPmzdq4caNOnDhRKPv18fHRp59+qn79+vFpAQDYBaPRqJdffll9+/bVqFGj9PPPP9/0vsLCwjRv3jzNmzdPZcuW1f33368uXbqoXbt2atSo0W1pcmVlZSkoKEg7duzQxo0btXXrVsVbmPLHWj169NCcOXO4uw4A7BjNOsAx1KldM8/lJ0+dKXDT6ZdfN2jHrr9yLOv3RF81ubPx9Z/zekJbQkKi4uLiVbZsGYuvcfp0sNLS0qw6nnW//KZdf+7JseyFEc/Kr2oVi9v6VvbRuLEv6/U3J+jS5Ss51gWfPXe9aXe78wcAcFwmR37zRqNRo0aNUp8+ffTWW29p6dKlBbrq59+ys7N18OBBHTx4UFOnTpXBYFD9+vXVvHlzNW7cWI0aNVK9evVUrVq1m751PiUlRaGhoTpx4oSOHj2qI0eOaO/evbc07WVeXFxcNHLkSI0fP55n1wEA7FJAQIDWrVunNWvW6LXXXtPp06dvaX9xcXFau3at1q5dK0kqU6aMmjdvriZNmqhRo0Zq1KiRatasKW9v75t+jcjISAUHB+vYsWMKCgrSwYMHFRgYqISEhELNTd26dTV16lT17NmTEwUA7FSzKka1r26kWQc4iDIeHqrmV1UXLuacOnHtz7+q4wPtrL7Y7Nq1OC3//gelpKTkWP7vnz3KeOS5/dlzoVY1p7bt2GX1e7t2LU7BZ3PelXf6TLBVTTtJcnVxUaOG9XM17RITk4otfwAAx2UiBVKVKlW0aNEijRkzRq+++qo2b95cKPvNzs7W8ePHdfz48RzLDQaDfH19VaVKFXl5ecnLy0tlypSRi4uLXFxclJ2drbS0NKWlpSkuLk4xMTGKjo7WpUuXFBERUeT56Nu3ryZPnqxatWpxcgAA7F6vXr3UrVs3zZs3T5MmTVJ0dHSh7Dc+Pl5btmzRli1bciwvXbq0/P395e3tLS8vL3l6eqpUqVJydXWVyWRSRkaGUlNTlZKSotjYWMXExCgiIkIXLlxQcnJykeaiYsWKmjBhgoYPH24TU30CAIrO4Kb8ngccTauWzXI1na6EheuvPYFqc29Lq/ax8oc1uRpMbqVL656md+VYVsbDQ0ajMdfF8WfPnrPYtAs5F6qt261v2tWqWSPXssD9f+uB9vdZvY+o6Jhcy/z9/YotfwAAx8Uo/QZNmjTRpk2btH37dr377rvaunVrkbxOdna2Ll++rMuXL9vMezcYDOrVq5fGjx+vpk2bcjIAAByKs7OzRo8erSFDhmjOnDmaNm2aoqKiiuS1kpOTdfLkSZ08edJm3r+3t7deffVVjRgxQh4eHpwQAAAAduihLp20Zu16paWn51i+aOm3quTtne8UkP/Yun2nNm7elmt5q5bN5OzsnGOZ0WiUZ/nyio7J2Qz7ad16NW1yh2pUD8jzNS5fCdPkT2YqOzvb6vdVu1bupt3ewAM6dDhId1lxV1/IuVAdPXY81/L6desUW/5ulpPJSQ926Vjk55Jf1So8zw4AighNuzy0b99eW7Zs0a5du/TJJ59o3bp1tzRtpi1zcXHR448/rtdee0133nknxQcAODQPDw+NHTtWL7zwghYsWKDZs2crJCTEbt9vjRo1NHr0aA0bNkxubm6cAAAAAHasbNky6vZgZ/20bn2O5VevXtM7736oYc88pQfat5XBkPOJdPEJCVrwxRL9tScw1z7d3NzU95G8p1Rvdk8T/b4x56wTSUlJeu/DT/T8sCGqX7+OyvzvgrH4+ARt3LxNa3/5VQkJiQV6X97eFVW1im+u6S3f/3iaevfspicefSTPWSSys7N14O9DmjV3gVJTcz4/r2oV31zP3rvd+bsZziaThg4eyMkOACUYTTsz2rZtq7Zt2+rs2bOaM2eOlixZopiYGLt4b1WqVNHQoUP1/PPPq3LlyhQbAIAbuLu766WXXtKYMWO0Zs0azZkzR1u3bi3QFb+2ymAwqEOHDho1apR69uxp9fM3AAAAUPL95/E+Onr8pE6fCc6xPCMjQ3Pnf6lFS79RgH81lSpVSs4mk8LCw3Xpcli+F7OPeG6IfCrl/czm+9rem6tpJ0lx8fGaPG2mJKlixQrKzMxUbOzVW3pfzw19WuMnfZRjWXZ2tn5c84t2/bFHNWtWV1XfyvLx8VZaWroiI6O0O3C/IiIi89zf4EH9iz1/AADHRNPOCjVr1tT06dP10Ucfae3atVq0aJE2bNigzMzMEvU+XF1d1bNnTz399NPq2rWrnJycKC4AAGYYjUY98sgjeuSRRxQSEqIlS5ZoyZIlOnfuXIl7LzVq1NBTTz2lp556StWrV6e4AAAADshkMunVF0dq4vuTdSUsPNf65OQUnTh52qp9dXuws1q1aJbv+vp166h1q+b6c3dgvjFRUXk/T9rk5KTHHu2t5StWW3UsDRvUU4f722nLth251kVGRSmyAFPf9+z+oJrcdUex5w8A4Ji4tLoAXF1d9dhjj2n9+vUKDw/Xl19+qW7dusnFxcVmj9nd3V2PPvqovv32W0VGRur7779Xt27daNgBAFBANWrU0MSJExUSEqK9e/dq7Nixqlu3rk0fc926dfXmm28qMDBQZ8+e1YQJE2jYAQAAOLgKFbz04aR3VL9enZva3snJSQP6PZbv3Wg3en7YENWsUbDxp4uLi0YMfybXM+UsGfTkE2rUsP4t5aZt61bq/59HbSZ/AADHw512Nz3AqaAhQ4ZoyJAhSkpK0vbt27Vx40Zt3rxZQUFBxfYMPGdnZzVp0kSdOnVS586d1aZNG5tuKgIAUBI1b95czZs310cffaTg4GBt3LhRGzdu1I4dOxRVgKt4C5u3t7fatWunzp07q3PnzqpZsybFAgAAsBOurq5KTk4plH2VKeOhd8e/qc1btuu7lT8oLi7e4jYGg0GNGzVQv8f7qm6dWla9TunSpfXBpLf1/aof9etvm5SSmmp2//e3a6P/PN5XFbw8FXr+QoHek4eHuya+/YZ++XWDvvluldLT063etlbNGho8qJ/q16trU/lD/sqWKUMSANglQ7Y9PJzFxsTHxyswMFC7d+/W4cOHdfToUZ08ebJAgwVrlCpVSg0aNFCjRo3UpEkTtWzZUvfcc49Kly5NEQAAKCbBwcHavXu3AgMDFRQUpKNHjyosLKzQX8fX11eNGjVS48aN1axZM7Vq1Uq1avHHPwCg5BoxYoTmzZtnMa5BgwY6duwYCYO8vb2tumBqwoQJmjhxIgnLR2pamg4fPqrA/X/r0qXLir16VUnJySpbpowqeHmpQgVPVfOrqratW6lCBa+bfp20tDQdPHREZ4JDdPVanOLi41TK1VVVfH1VpUpl1apZQ76VfQrlPSUmJur4iVMKOnZCx46fVFR0tFKSU5SWnq7y5crJp5K3fHwqycfHWzUC/NW82d0yGAw2nT8AgGOgaXebpKenKzQ0VOfPn1doaKguXryo6OhoxcTEKDY2VsnJyUpLS1NaWpoMBoNcXFzk4uKi0qVLy8vLS15eXqpYsaKqVasmf39/BQQEyN/fX0YjM5wCAGDrYmNjde7cOYWGhio0NFTh4eGKiYlRTEyM4uLilJqaqrS0NGVkZMhkMsnFxUWurq4qW7asvLy8VKFCBfn4+FwfA9SoUUPly5cnsQAAu0LTDgVF0w4AANgbpse8TZydnVW7dm3Vrl2bZAAA4GA8PT3l6emppk2bkgwAAAAAAADkidu0AAAAAAAAAAAAgGJG0w4AAAAAAAAAAAAoZjTtAAAAAAAAAAAAgGJG0w4AAAAAAAAAAAAoZjTtAAAAAAAAAAAAgGJmIgUAAAAAAKC4BQcHWxUXHh6ul156iYRBCQkJVsWdOXOGZAEAgBKBph0AAAAAACh2ly5dsiouJiZGM2bMIGGw2sWLF0kCAAAoEZgeEwAAAAAAAAAAAChmNO0AAAAAAAAAAACAYkbTDgAAAAAAAAAAAChmNO0AAAAAAAAAAACAYkbTDgAAAAAAAAAAAChmNO0AAAAAAAAAAACAYkbTDgAAAAAAAAAAAChmNO0AAAAAAAAAAACAYkbTDgAAAAAAAAAAAChmJlIAAAAAAACKm7+/v44ePWoxrkKFCho+fDgJg6ZPn67k5GSLcQEBASQLAACUCDTtAAAAAABAsatevbpVcZUqVdL7779PwqD58+db1bSrUaMGyQIAACUC02MCAAAAAAAAAAAAxYw77QAAAAAAAADAwZ0JDtHlK1eu/9y2dSsZjTd3z8fxE6cUGRUlSTIYDLqvzb02cfzFcVwoPNQPN9qx609duxYnSfL2rqhWLZrZxfuiaQcAAAAAAADArm3asl07dv0pSXJyctKEt14nKf+yYtWP+vvg4es/39uqxU037RYt/VZnQ85Jun3NFWuOvziOyx5s3/mngs+GSJKa3nWHmja5s1iOg/rhRj+v33D9fHBzc6NpBwAAAAAAAAC27lzoBX2xaJkyMjIkSSYnJ5Jyk5KTk/Xjml+UkZkpSapVs7ra3NuSxNixrKwsLft2ha5evSZJcnd3K/SmHecV8P9o2gEAAAAAAACwS6lpaZoxe971hh1uzbW4eP2w5ufrP7e/rzXNFTsXdPT49YYd5xVQ9GjaAQAAAAAAALBLS5Yt18VLl0nEbeZb2UeZmf9tlN7sFJuOdFy26lpcnBZ8tZT6AbcRTTsAAAAAAAAAdmdv4H5t2LSVRBSDl0Y/z3GVYPEJCfprd6B++XWDwsLCqR9wG9G0AwAAAAAAAGBXomNiNXfBVyQCsNLW7bu068/dunw5TFHR0crOziYpQDGgaQcAAAAAAADYsNTU1OvPZHNyMqlUKVeL2yQmJUn/+9Ld1dVVJlP+XwOmpaXpbMg5XQmLUFh4uNLT0lW1ahX5V/OTn18VlS5V6qaOOzw8QmdDQnXp8hW5u7vJ27uiGtavJze30kWar6ysLM36bL4SEhIlSc7OzsrMzFRWVlahvk56RobSUlMlSQaD8fr7ys7O1snTZ3Tp0mXFxF5V+XLl5FvZRw3q15WTk1PuWiUm6uTpYIWcC1UFLy/VqV1TVXwry2AwFNs5kd97TUxMyrE8IyNTiYn/zbPRyemmz5V/JKek6EjQMYWHRyouPk5Vq1RRnVo15OtbuVinQ7yVczkjI0Op/ztPjEajSpf+7zbp6ek6eeqMQs6dV1p6mtrf11oVK6wOPNcAACAASURBVFS4LZ/NvBw9dlyHDgfd1rwW9nlVnJ9JW/xdWNivnZ6RoTNnzioiMkoRkZFKT8+Ql2d5eXl5yreyj6r5VS3wPjOzsnTp0hUFnw3R2ZBzyszMlE8lb/n4VFLd2rXk5eV5U/s8evS4roSFKzomVmXLeKh27ZqqUT1Ari4uNv9vPk07AAAAAAAAwIa9Pm6iLl2+IkkqX76cvpg302z83n0HNGXarOs/jxn1nO5rc2+uuKysLG3f+YeWf/+DYmJi891fi+Z3a+Rzz8jd3d3isSYmJenLRV9r34GDSkpKyrXexdlZLZrfrSce6yPfyj5Fkq+f1q7X0WMnrv88sP/j2rx1h0LPXyjU15kw6SOdOh0sSfL0LK+Fc2fo4OEgLf36O52/cDFXfOXKPnrmqSfVtMmdkqQ9e/fru5WrdeFi7mfuubm5qU+v7urds/ttPSfy89GUT3X4yNFcy//4a4/++GuPJKl+vTp6f+JbkqRx49/T2ZBQSZKXl6fmzpx6fZvk5GQNHvbC9Z8f79tbvXt208of1mj9bxtzNXAkycPdXc8+M0ht7m15SzUzd1xFdS5PmzlXgfsOSJJ8Knnrs5lTdep0sD77/IvrNZSk2jVrXG/aFcVn0xYV5nlV3J9JW/pdWNivHR+foN82btbvG7bo6rVr+cbVrFFdnTverw4PtJOTFU327Tv/1BeLlio5OSXP9SaTSZ06tFefXg9b3bzbun2nvl+1RpFRUbnWGY1GNahXV6++PEplPDxs9nNB0w4AAAAAAACwYWlp6Xn+f37+uUPFnMysLH005VMdPHTEYuzewAM6d+68Xh4zUrVr1cg3LvjsOU2b+ZkiIiLzfy/p6dr15x7tDTygpwb2U9fOHQo1V6fPnNV3K3+4/nOTu+7QQ107afPWHYVel4yMzBw/794TqE9nf67MzMw848PCwjV1+mx9/MEE/bk7UKt+WJPvvpOSkvT18pVycjKpR/eut+WcKMztMzIyr98JGBmZ88vz7GxdXydJEZGRmvXZfO36c0+++0tITNSns+bpSNBxDX6q/03fLWPuuIrqXL7xvUrSkaBj+mDy9FzLi/KzaY3297VR7Vo1cy0/HHRUewMPFMnvtsI8r4r7M2krvwsL+7WvhIVr0gdT82yC/dvZkHOa/8ViHT9xSi+MeDbfOxPT0tL05eKvLf5ezsjI0G8bNmv33n365KNJKl++nNn4b79bpR/W/Jzv+qysLB09fkKTPpiqCW+9Lg8P22x207QDAAAAAAAAHMy3363K1RTw8iwvH59KSktLU3h4pBJu+EI9IjJK734wWZ/NmKqyZcvk2t/ewP2aPnOuMv715bjRYJC7h7vi4xNyLE9LT9fCr5bKZDKp4wPtCuU9JScna8bsedenwSxTxkMjhz9z01PaFURs7FVNnzXP4hScaenpev3NCbnylJ8lXy9XtWpV1eTOxnZ7Lm7asj3HzyYnJ3lV8FJSUtL1KU7/P3abJGn4s08X2fEU5bkcFR2jydNm5duwK4rPprXuaNxQdzRumPucTUsrsqadPX4mi/N3YVG89iefzsnVsHN1dVEVX1+VLl1KsbFXdSUsPMf6Hbv+lK+vjx7r0yvPfX445VMFHT2ea3np0qVUxsNDUVHRyrrhmYpXr17TjDmfa/y41/KdJjcpKSlXw87k5CSj0ai09JwXNoScC9XkaTP13oRxNnnu0rQDAAAAAAAAHEhqaqrW/7bx+s/Ozs4aPWKYWrVslqPBdfBwkBZ8ueT6HRvJySn6ad16DXryiVz7+2Lx1zm+KK5YwUvDnnlKjRrWl6urq2JiYnXq9BktXrZcUdEx1+O+WLRMrVo2k7ub2y2/r4VfLVP4DXeXPD9siDzLl79tef2nOdC2dUs90uth+flVVWxsrH5c84t+37jletyNearmV0UD+j2uunVrKz4+QYH7DuiXXzcoJvbq9Zidu/4s9qbdxLffUFZWlsIjIvX6uInXl7dofrdGDHtGkuTkdGvPnCvl6qon+z2mTh3ay9nZWZJ0JjhEy75ZoaPH/3+60y1bt6vbg53kX82vSD4bRXkuZ2ZmXr/ry+TkpEYN6ysgwF/ly5VVndq1Cv2zaeuK+ry63Z/J4vxdWBSvffDQkRzTCjs7O2vQk0/owS4dc5yPl6+Eacmy5dr/96Hry/btP5hn027vvgO5Gnbt2t6rTh3vV706teXk5KTU1FStWPmj1q3/Xdn/a94FHT2u3Xv3qXWrFhZz0eSuO/6vvTsPr7K88wb+y8kKibKEEJawiKwqFSqiBcfRSnGpvNjWOjpOtdp26lartWrl1dGpL3WK2qnLjIPYinUcnBk743TeV22rjjqt1YIOFgiIymYEYtgMIQmQ5f3DNuWQHZKcQD6f68p18dznfpb8zv2c6/B889xPXHTBF2LE8KJIJBKxcdPmePSxJ+KtfaZiXblqdWwu/TAGFQ7sdudFIgAAAACAHmPZ8uLYu8+dB+eeMzM+dfKJje5Im/SJ4+I73/5mUltTd0f8x38+m/TcrbFjjo57v39nfHLy8ZGdnR0RHz9/6uSTToy//qtbYsCA/Ia+e/fujV/9+rWD/p1e+dWr8cqvXm1YnvHp02LqlE92eW0vu+TP47pvXBkjhg+L9EQiBuTnx9cuvyQ+MfHYRn2PO3ZC3H3Xd+OET06KI/LyYsjgQTF71jlx3TeuTOq3ctXqlI+ZXr16RW5ubvTe76J+r5ycyMvLjby83OjVq9cBbz89PT3m3PytOPvMGQ2BXUTE6KOPilvnfDupfnX19fHY44s65ffsqrFcOLAg5n73trhtzo1xycV/Fv/r3LOjd+9eHX5udnedPa66+pxM5WdhZ+z77XfeTVq+6ILPx9lnzmg0HocMHhTfvv6aGDiwoKFt3foNsWfPnqR+tbW18Y+L/jWp7RMTj41rrvxaHDN+XKSnp0dERHZ2dlzyFxfGn0w/ud1j/MIvfj5u/c4NcfSokZGRkRGJRCKKhg6J79x0fYwZnTz961u/W94tzwuhHQAAAAD0IPtPZZaW1vwlwuHDipKelbW59MOk17dv3xFP/+czDcsZ6elx9de/Erm5TT8rqHBgQdz0rWuT2v7r5f8+qN+ntPTDWPDjnzQsDx5UGF/+0kVdXtejRg6Ps8+c0eRrJ0w+Pmk5Iz09vnb5JZGR0XgitGMmjIuc319kj4j4qLz8sB+Tnz7tT+KYCeOafC0zIyO+dvklDRf0IyLeWraixWfSHYiuGstZmZlxx23fiaNHjezUc5OuPSdT+VnYWfvevDl5TE3/1EnNHkNmZmaM3ScUq62tjV27KpP6vPTKr2Pjxk0Ny/n9+8f137iy2Skvzz0n+dmBy4tXtViH4cOK4nOzP9vs58ipp0xrVLfuyPSYAAAAANCD9O3bJ2n5uZ8/H5OPnxgTxo9tsv/3vntbwxRl+1u2ojjpborT/vSUGDp0SIv7H3XUiDh61FHx3pq1EfHxFIilpR9G4QFMU1ZbWxs/fHB+VFVVR8THd2x985orIicnu8vr+rnZ5zZ78Xn/mo8aNTKGDhnc7LYGDMiPkg82RkTE7t17Ym9NTWRmHL6Xcs856zMtvj54UGGMGzs6ile+3dC2qbQ0CgoGdNgxdNVYnnXuWVGwz11OnXVu0rXnZCo/Cztr358/79yYdPzH04DmZOdEfn7/5j+L6+qibMu2Fve56u3kOxTPOP3UOOKIvBaOcWRMPPaYWPn79SoqKlr8LPz8eecmhftNfY7sq3znzm45boV2AAAAANCDHDthfNJyZVVV3PbX34sJ48bGn546PSYfPzHp4mxzF70jIjZuSr4zaMzoo9t0DOPGjm64WBwR8cGmzQcU2v3zU0/HO+++17B8wRdmJ9191JWGDG7+gv++d+lERKvPUdp/+rk/PJvrcHTkkUfEsKKhrfYbVjQ0KbTbvPnD+MRxx3bYcXTVWJ58/MQuOTfp2nMylZ+FnbXvYUVDWzw3q6qq4sOyLbFpc2n89N9/FmvXbWhxf/vfDXrCJye1eoy333pTO97vQS2+npeXfOfh7t27u+W4FdoBAAAAQA/Sv3+/+PRpp8aLL72S1L7y7dUNdzQMHFgQnzju2PjkpIkx8bhjmn2u1KZNm5OWH3/in+OZ537Z6jFs3G+9A5mmbEXxqvj3//i/DcsTxo2Nz80+N2V1LRjQv819BwwYYCD+3tFHjWxTv/3Dg46eDrKrxnLR0KFdcm7StedkKj8Lu2LfG97/IFYUr4yVq96OzaVl8WFZWVRU7GrXcZbuc86mpaXFUSOHd+z73cqdt4lE+iExboV2AAAAAHAYqatrfbq8K7725UhPT8QvX3ipydc//LAsnn/xpXj+xZciPT09pp08NWbPOidGjhiW1G/jpk1JyxW7dkXFrl3tPua9e/e2q39Fxa647+/mN0wN2KtXTnzj6q+l7M6j7OysZp8f1ZTMjPRuNyZSpaXp8fZVtN90f9u2be/Q4+iqsZzdytStHXVu9nRdfU6m6rOws/f9wQcb4+Ef/SRWrFx1UPXZs2dPbNsnFMzLy+3Qz+vs7Kw4Ii/vsBi7QjsAAAAAOIxUVVW12ieRSMTXv/rlOPMzZ8T/e/bnseTNpbFzZ0WTfWtra+O/f/2b+M1rv41b53w7jjtmQsNr5eUVHXLM7Z3+cc26dUmhTVVVdfzDgoXN9l+/4f2Gf9fU1sZ3v3d3w/KA/Py46uuXH9Txp6dnHPJjIlWysrLa1jF5dsLIzOzYmqdqLHfWudnTdfU5mcrx01n7Xra8OOb+zb1RU1vb7DoZ6ekxfPiwOGbCuChe+XasWbuuyX47Pvooabmj7xDt7p/B7SG0AwAAAIDDSMWuyjb3HTliWFx9xVejrq4u3n1vbawoXhnLi1fFylWrY8+ePUl9a2prY94998d9994V/fr1jYiIgQUDYuu2bQ19vnrZl2L48KJ2H/PoUQf/HLrfLVtxQH2HDhlsTKRQaWlZm/pt3Zp8Z11HX/TvTmO5I85NulYqx09n7Hvnzoq478H5SYFdWlpaTDp+YnziuGNixPBhUVg4MAbk94/09I/vUrz/7x5uNrTr26dPpKWlNdwdXf5RuUHTDKEdAAAAABxGSg/gWV+JRCLGjjk6xo45Oj43+9zYu3dvrCheFS++9Eq8+trihn6VVVVRvGp1TP/U1IiIGDx4UMOztiIi+vbtE8eMH+dNOAzGRFdZ//77beq3deu2pOX9p8s8WN11LB/ouUnXSuX46Yx9L3lzadLdcdnZWXHLTde3eDfnzp07m30tKysrBuTnR9mWLRERUb17d+zatavVKUxXvb06Nm4qbViePu2kyG7r3bmHKKEdAAAAAHRjaftMC1hVVRXV1bsjp4XnYi0vXtni9h77xydj9+7dERExYviwOPMzn27UJzMzMyYdPzEmHT8xBg/+afz03/+z4bX31qxtCAaGDhmUtN7adevj5KlTWv2dnn/x5YY7MjIzM+PP/+wLkZ2d3eaa5Pfv3+RxN+f1xW/Ejh1/vAC977p9+/bp8WMilcrLd8ZH5eXR58gjW+z3xptLk5YnHtuxU0Gmaix31rlJ10rl+OmMfa9c9XZS3/M/979aDOxqa2tj5dvvtLi/IYMLG0K7iIjlK1bFSVNPaHGdf1iwMEo+2BgRETk5OXHqKZ867MeS0A4AAAAAurHc3NyIso8vdNbX18e69etj/LixTfZ96ZVfx/btO1rc3muvL2m4cJqb2zvOOP3UyMho/jLh1CknJAUD+07NN2Rw8tSSv3z+pfj87HNbvOhcVrYlHnn08aipqYmIiLy83PjSn1/QrpoMHTI4vnb5JW3uv2FDSUNol5Ge3q51e8KYSLXXXl/SYghbvPLtePuddxuW+/fvF4MHD+rQY0jVWO6sc5Oulcrx0xn73r4j+Rl0rZ1vb7/zblRXV7fYZ8TwYfHWPlMTP/fLF1oM7dasXdcQ2EVETDzumBbPh8NFwukEAAAAAN3X/neCvfbbN5rst279+/Hwjx5rdXtHHz2y4d+7dlXG8hUt34W1/zOKRh31x/UnT5oYQ/Z5Jlz5zp3xH//5bLPbqquri5888c8NF4ojIqadPLVHXIjtzmPiQJXvrOiQ7Tz6k3+Kd99b0+Rruyor4/F/+uekto6+y667jOWOPDcPZR01rrpSKsdPZ+y7YEB+0jplZVub3d7adevjh/c/1OpxfvacMyMrM7Nhedny4vjtkjeb7FtTUxMPPfxo8u95/MQe8fkutAMAAACgSY8//njcddddST/7XuSja4wfOzpp+f8+8/P46b//LMrLP35+0MZNm+O5X7wQ8+69r0132hw7YXzS8v1/93AsXvI/jfrt3rMnfv2b38ajP/mnpPYxo0c1/DsjIyO++uW/SHr9X376dCz48U9i735jpWzL1rj9zr+J37z+x+dwZWVmxtlnzvAmp3hMtFWvnJyk5f9Z+rtYXrwyampqoq6u7oC3W1NTE3f/4MF483/eiqp97tZ5v+SDmHPbnfHOu2uSxsysz57d4TXtDmO5I8/NQ0lnjauulMrx0xn7HjlieNLyk//60/jd8hWNxuGLL/133HbH92JbE3fzflSe/Iy7/P794pyzZya13fO3D8YL//Vy0ufUh2Vlcedd98Tadev/OEZ65cSJJ0zuEZ/v/oQFAAAAgCZdc801UV5entR2/fXXuyuqi50y/VPxrz/9j6iprW1oW/Qv/xaL/uXfolevnKiqqm7X9j5zxmnxi+dfjPdLPp52rHznzvj+vffFiOHDYuiQwdGnz5Gxdt36ePfdNUn7/MO6w4qGJrV9YuKxcfJJU+K115c0tP38ly/GC//1SgwfVhRpaWlRWVkZm0s/jPr6+qR1L7v04kbbo+vHRFv16XNkZGdnxe7df7zAfsed34+IiBOnfDJuvuHaA9721m3b4nvz/jbS09Nj5IhhsWNHeWzdtq1Rv69c9qUYOWJYp/x+qR7LHX1uHio6c1x1pVSOn47e94lTJsc/PflUVOzaFRERu3fvie/OvTtGH31UDCocGOU7K+Kdd99r8bPm/9x1TwwfXhRf/8qlUVg4MCIiPjf7s/HSy7+KHR99PP1mXV1dPPTwo/Hwj34Sw4qGRvXu3VHaxDFefcVXo1+/vj3i892ddgAAAAActLq6uigvL0/6qaioUJgOUDiwIG7+9jeTphX7g/0vmPbu1Suuv/bKFreXkZERX//aZY3C1/Ub3o9XX/ttPPvz52PV2+80CgWGFQ2JL3/poia3edVffiVOPWVaUltNTU2sWbsu3luzNjZtLm10EXbWZ8+Kz5xxmje4G4yJ9viT6Z9qsn3nzp0HtL3hw4oiv3//huXa2tp4b826JgO70//0lDjj9FM7tbapHMudcW4eKjp6XKVKKsdPR+67X9++8ZdfvbRR+7vvrY1fvfp6/G7ZiqTPmgnjxsa87/11Ut8dH30Uv1u2IjZtLm1oy+3dO/5m7l/F6KOPSupbW1sb69ZviM1NHOPnZn82Tp46pcd8vgvtAAAAADhoy5Ytiz59+iT9nHjiiQrTQSZP+kTM+c63on8LdxqcMu3kuPf7d8aokSNa3d74sWPi+3NvjxHDW79jKTs7K/7s/PPib+beEdnZ2U326d27V1x79V/G9ddeGb17925xe0MGD4qbb/hmXPoXF6akljn7TcV3oDIy0lN6DB09JvYNirIyMyORaPrS8UUXfCHGjR3TYbUZPKgw7rzjljhq5PBm+/Tr1zeuvuIrcdXXv3JQx9+W4+rosdzWunbWuXmo6IhxlepzMtWfhR2972knT42bb/hmUqi+v/79+8VXL/tSfPf2W2LUUSPinLM+0+pxDsjPjztvnxOfOeO0yEhv/j0bP25MzP3rW+PiC7/Y4e93d5ZWv39sCQAAANDFrrrqqnjooYda7TdhwoQoLi5WsC7Sp0+fRtNjVlVVNXlx86233opJkyYltY0fPz5WrlzZKcdWUFAQW7ZsabXf7bffHnfcccdh9b5s374j3lu7Ltat2xDp6YkYOmRwjBg+rGH6sfaoq6uLdes3xIriVfHemrVRsasyampqYkB+/ygcWBCFhQPjuGMmRP/+/dq8zerq3bFu/fpYs/bjn+rd1dG/X7/Iz+8fx4wfd8g+d6unjIm22lz6YWzcuCnq6+sjNy83hhUNjdxWgoKIiMrKqrjkK3+88++kE0+IG7/1jYiIWFG8KopXvR1btmyNI/LyIje3dxQNHRoTjzsmcnK6PpRK5VjujHPzUHCg46o7SuX46ch9V1VXx9Kly+L9kg9ic2lpZGdnx4D8/jFmzNFx3DETGgXRa9aui7Xr1kddXX0MLBgQE8aPjaysrCa3vbemJtavf7/hTsB+/frG4EGFMWTwoCgaOqRHfp4L7QAAAICUE9p1T0I7oKO1FNoB9HSmxwQAAAAAAIAUE9oBAAAAAABAigntAAAAAAAAIMUylAAAAACgdfX19VFSUhLvvPNObN++PYqKimLUqFFRUFDQ6ro7d+6MtWvXRklJSVRVVcXAgQNj8ODBMWrUqEgk/E01AABCOwAAAOhxakrej4yiYQrxexUVFXHuuecmtc2cOTPmzJkTERFbt26NH/7wh/Hggw/Gjh07Gq1fVFQUt956a1x++eWRmZmZ9NozzzwT8+fPj2effTb27t3b5LoXXXRR3HzzzZGfn9/ica5fvz4uvfTSpLZx48bF/Pnz2/y7Lly4MBYuXNiwnEgk4t57743JkycfUO1uvPHGWLx4cUMd97dhw4Y47bTTktpeeOGFSE9PN/AAAPYjtAMAAIAeZuf9P4j0/Pzodd75kTlufI+vR01NTbz88stJbSUlJTFnzpx46qmn4rLLLmsykNq37xVXXBH3339/vPjii1FYWBgfffRRXH311fHEE0+0uO+SkpK4++67Y9GiRfHUU0/FSSed1GzfysrKRse5cuXKdoV2a9eubbSN55577oBDu7feeqvR9lo75vr6eich9GCJRCKGFQ1tWB4zepSiAPye0A4AAAB6oL0rlsfeFcsj89jjhHfNmD9/flx55ZVtDpmKi4tj1qxZ8fzzz8eMGTNiyZIlbd5XSUlJzJo1K5YtWxaFhYWKDxy2cnKy42/vnqsQAE0Q2gEAAEAPJrxr2nvvvRdXXXVVu+8KW7x4cfTt2/eA7iYrKyuL6667LhYtWnTI1Mk0lwAAHUdoBwAAAAjvmlBXV9fw79zc3LjoootiypQp0adPn1i2bFk8+eSTsWbNmkbr7R/YZWZmxvnnnx+TJ0+OoqKiKCkpiYULF0ZxcXGjdZ9++umoqKiIvLy8Q6JG9913X/ziF7+IiI+nynzkkUeSXi8sLIzbbrstqU3QBwDQNKEdAAAA0EB419ipp54ajz76aIwa9cfnLl144YVx7bXXxowZM2L58uXNrnvSSSfFggULYuLEiUntN9xwQ1x33XXxwAMPJLVXV1fHs88+G1/84hcPidqMHTs2xo4dGxFNh3b9+vWLq6++2okFANAGCSUAAAAA9rd3xfIon3tHlH///8Tet1f12Dqcfvrp8dJLLyUFdn9QWFgYCxYsaHbdMWPGxMsvv9wosIuISCQScc8998To0aMbvbZs2TIDEACgB3KnHQAAANCsnnznXSKRiB/84AeRlpbWbJ+pU6dG//79Y9u2bY1ee+CBByI7O7vZdbOysmLatGnx7rvvJrWXlZUZeAAAPZDQDgAAAGhVTwzvLrnkkpg0aVKLfRKJRAwdOrRRaDd16tQ488wzW93HH6aW3JfQDgCgZxLaAQAAAG3Wk8K7s846q039+vXr16ht/Pi21SUvL69RW2VlpYEGANADCe0AAACAdusJ4d3w4cPb1C8nJ6dRW1PPwAMAgJYI7QAAAIADdjiHdyNGjDjgdYV2AAC0l9AOAAAAOGiHW3iXmZkZgwYNOuD1e/fubVAAANAuQjsAAACgwxwu4V2/fv0ikUgcdu/Pnj17DFIAgG5KaAcAAAB0uJ7wzLtD0fbt2xUBAKCbEtoBAAAAnUZ4170I7QAAui+hHQAAAHSw+j17Ys/i17vv8VVVdvk+hXfdQ2lpqSIAAHRTQjsAAADoYHUVFVEx/+8UognCu45VW1vb5r579uyJxYsXKxoAQDcltAMAAAC63P7hHa1LS0tr1LZ169bYsmVLDBgwoNX1X3vttaisrFRIAIBuKqEEAAAAQKrsXbE8yufeEXv+Z4litCI/P7/J9jfffLNN6z/22GOKCADQjQntAAAAgJSr27ZNEVqRn58fGRmNJ0164403Wl136dKlsXDhQkUEAOjGhHYAAAAAh4BEIhGDBw9u1D5v3rxYunRps+utXr06Zs+eHXV1dZ16fE1N3/nBBx90+n4BAA6b73tKAAAAAHBomDVrVqO2HTt2xMyZM+NnP/tZbNvnjsWtW7fGXXfdFdOmTYsNGzZ0+rENHDiwUdvOnTvjlltuibVr10ZdXZ1n6gEAtEBoBwAAAHCIuPjii5tsLysri9mzZ8eAAQNi5MiRMXTo0CgoKIg5c+bE1q1bu+TYCgsLIzs7u1H7vHnzYtSoUZGenh65ublRU1PjjQQAaILQDgAAAEiptD59ImP0WIVog2nTpsUFF1zQ7Ov19fWxfv362LhxY9TX1ye9lpWVFXPnzu289zEtLWbOnOlNAgA4QEI7AAAAICXS+vSJ3hf9RfS794HIGDFSQdrokUceiRNOOKFd6/Tu3Tt+/OMfx/Tp0zv12G666aYmn20HAEDrhHYAAABAl9o3rOt19rmRlpWV0uPJyMiIrBQfw74KCgpafP2II46IV199NW655ZbIy8trsW8ikYjLLrssVq9eHRdffHH079+/XceSm5vbrv6nnHJKLFq0KPr27WugAwC093ty/f5zJQAAAAAHpb6+PuorK7vt8X10+5yo+7C0y/ebdmSf6PXZWZFzxsxGQd1VV10VDz30UKvbmDBhQhQXFxtkv1ddXR3PPfdcLF68OEpLPihIwgAADo9JREFUS6OsrCxyc3Nj3LhxMW7cuJgyZUqMHj26y4+rpqYmXn/99Vi9enVUV1dHbm5uDBgwICZNmhRDhgzpkH0UFBTEli1bWu13++23xx133GGwAADdXoYSAAAAQMdKS0uLtHbeodSlEl078U5LYR0HJycnJ84777w477zzutVxZWRkxPTp0zt9Ok4AgMOJ0A4AAADoFMI6AABoO6EdAAAA0KGEdQAA0H5COwAAAKBDCOsAAODACe0AAACAgyKsAwCAgye0AwAAAA6IsA4AADqO0A4AAABoF2EdAAB0PKEdAAAA0CbCOgAA6DxCOwAAAKBFwjoAAOh8QjsAAACgScI6AADoOkI7AAAAIImwDgAAup7QDgAAAIgIYR0AAKSS0A4AAAB6OGEdAACkntAOAAAAeihhHQAAdB9COwAAAOhhEn36RM7pZwjrAACgGxHaAQAAQA9z5C1/FWmJhEIAAEA34hs6AAAA9DACOwAA6H58SwcAAAAAAIAUE9oBAAAAAABAigntAAAAAAAAIMWEdgAAAAAAAJBiQjsAAAAAAABIMaEdAAAAAAAApJjQDgAAAAAAAFJMaAcAAAAAAAAplqEEAAAAQKpVVla2qd/q1atjyJAhCkZs3bq1Q8cWAECqCe0AAACAlFuyZEmb+tXW1samTZsUjDb77W9/qwgAwCHB9JgAAAAAAACQYkI7AAAAAAAASDGhHQAAAAAAAKSY0A4AAAAAAABSTGgHAAAAAAAAKSa0AwAAAAAAgBQT2gEAAAAAAECKCe0AAAAAAAAgxYR2AAAAQMoVFRUpAp1i2LBhigAAHBIylAAAAABItVGjRrWp3/Dhw+NHP/qRghFf+MIXory8vNV+Rx99tGIBAIcEoR0AAABwyMjNzY0ZM2YoBJGVlaUIAMBhxfSYAAAAAAAAkGJCOwAAAAAAAEgxoR0AAAAAAACkmNAOAAAAAAAAUkxoBwAAAAAAACkmtAMAAAAAAIAUE9oBAAAAAABAigntAAAAAAAAIMWEdgAAAAAAAJBiQjsAAAAAAABIMaEdAAAAAAAApJjQDgAAAAAAAFJMaAcAAAAAAAApJrQDAAAAAACAFBPaAQAAAAAAQIoJ7QAAAAAAACDFhHYAAAAAAACQYkI7AAAAAAAASDGhHQAAAAAAAKSY0A4AAAAAAABSTGgHAAAAAAAAKSa0AwAAAAAAgBQT2gEAAAAAAECKCe0AAAAAAAAgxYR2AAAAAAAAkGJCOwAAAAAAAEixDCUAAAAAoCmPP/54lJSUJLXdeOONkZHhkhIAQEfzDQsAAACAJl1zzTVRXl6e1Hb99dcL7QAAOoFvWAAAAAActLq6uqioqEhqSyQSkZeXpzgAAG3gmXYAAAAAHLRly5ZFnz59kn5OPPFEhQEAaCOhHQAAAAAAAKSY0A4AAAAAAABSTGgHAAAAAAAAKSa0AwAAAAAAgBQT2gEAAAAAAECKZSgBAAAAQOvq6+ujpKQk3nnnndi+fXsUFRXFqFGjoqCgoNV1d+7cGWvXro2SkpKoqqqKgQMHxuDBg2PUqFGRSPibagAAhHYAAABAD1dRURHnnntuUtvMmTNjzpw5ERGxdevW+OEPfxgPPvhg7Nixo9H6RUVFceutt8bll18emZmZSa8988wzMX/+/Hj22Wdj7969Ta570UUXxc033xz5+fktHuf69evj0ksvTWobN25czJ8/v82/68KFC2PhwoUNy4lEIu69996YPHnyAdXuxhtvjMWLFzfUcX8bNmyI0047LanthRdeiPT0dAMPAGA/QjsAAACgR6upqYmXX345qa2kpCTmzJkTTz31VFx22WVNBlL79r3iiivi/vvvjxdffDEKCwvjo48+iquvvjqeeOKJFvddUlISd999dyxatCieeuqpOOmkk5rtW1lZ2eg4V65c2a7Qbu3atY228dxzzx1waPfWW2812l5rx1xfX2/QAQA0wfwLAAAAAE2YP39+XHDBBS0GdvsqLi6OWbNmRXl5ecyYMaPVwG5fJSUlMWvWrCgtLVV4AIAeSmgHAAAAsJ/33nsvrrrqqnbfFbZ48eLo27dvLFmypN37LCsri+uuu+6QqpNpLgEAOo7pMQEAAACaUFdX1/Dv3NzcuOiii2LKlCnRp0+fWLZsWTz55JOxZs2aRuvtH/RlZmbG+eefH5MnT46ioqIoKSmJhQsXRnFxcaN1n3766aioqIi8vLxDokb33Xdf/OIXv4iIj6fKfOSRR5JeLywsjNtuuy2pTdAHANA0oR0AAAD0MJVba6J3vksCbXXqqafGo48+GqNGjWpou/DCC+Paa6+NGTNmxPLly5td96STTooFCxbExIkTk9pvuOGGuO666+KBBx5Iaq+uro5nn302vvjFLx4StRk7dmyMHTs2IpoO7fr16xdXX321QQQA0AamxwQAAIAe5vUHy2LJgi1R/sEexWjF6aefHi+99FJSYPcHhYWFsWDBgmbXHTNmTLz88suNAruIiEQiEffcc0+MHj260WvLli1TeACAHkhoBwAAAD3Q5qVV8cr3SoV3LUgkEvGDH/wg0tLSmu0zderU6N+/f5OvPfDAA5Gdnd3sullZWTFt2rRG7WVlZYoPANADmQsDAAAAerDNS6ti89KqGDSpV4w958g4cmiWovzeJZdcEpMmTWqxTyKRiKFDh8a2bduS2qdOnRpnnnlmq/v4w9SS+xLaAQD0TEI7AAAAQHjXhLPOOqtN/fr169eobfz48W1aNy8vr1FbZWWlAQkA0AMJ7QAAAIAGwrs/Gj58eJv65eTkNGpr6hl4AADQEqEdAAAA0IjwLmLEiBEHvK7QDgCA9hLaAQAAAM3qqeFdZmZmDBo06IDX7927t8EDAEC7CO0AAACAVvW08K5fv36RSCQOu99rz549BjMAQDcltAMAAADazLSZh7bt27crAgBANyW0AwAAANpNeHdoEtoBAHRfQjsAAADggAnvDi2lpaWKAADQTQntAAAAgIMmvEuN2traNvfds2dPLF68WNEAALopoR0AAADQYYR3nSctLa1R29atW2PLli0xYMCAVtd/7bXXorKyUiEBALqphBIAAAAAHW3z0qp45XulsWTBlij/YI+CdID8/Pwm29988802rf/YY48pIgBANya0AwAAADqN8K7j5OfnR0ZG40mT3njjjVbXXbp0aSxcuFARAQC6MdNjAgAAQAer3VsfW1ZVd9vjq9ld1+X7NG3mwUskEjF48OB4//33k9rnzZsXZ599dkyaNKnJ9VavXh2zZ8+OurrOfd+bmr7zgw8+iLq6ukgk/N04AEBrhHYAAADQwfbsqo3F/7BFIZogvDs4s2bNir//+79PatuxY0fMnDkzHnnkkTjllFOif//+EfHx8+4efvjhuPfee2Pr1q2dfmwDBw5s1LZz58645ZZb4oorrogRI0ZEdXV19O7d2xsJANAEoR0AAADQ5fYP72ibiy++uFFoFxFRVlYWs2fPjrS0tBg+fHjs3bs3Nm3aFPX19V12bIWFhZGdnR27d+9Oap83b17MmzevYXnv3r1NTvMJANDTmZsAAAAASJk/PPPuw+IqxWiDadOmxQUXXNDs6/X19bF+/frYuHFjo8AuKysr5s6d22nHlpaWFjNnzvQmAQAcIKEdAAAAkHKVW2oVoY0eeeSROOGEE9q1Tu/evePHP/5xTJ8+vVOP7aabbmry2XYAALROaAcAAAD0aBkZGZGV1X2erVdQUNDi60cccUS8+uqrccstt0ReXl6LfROJRFx22WWxevXquPjiixued9dWubm57ep/yimnxKJFi6Jv374GFgBAO6XVd+Xk5gAAANADVO2oiRf+9yaFaIe/f/l/x7MrHm+134QJE6K4uFjBfq+6ujqee+65WLx4cZSWlkZZWVnk5ubGuHHjYty4cTFlypQYPXp0lx9XTU1NvP7667F69eqorq6O3NzcGDBgQEyaNCmGDBnSIfsoKCiILVu2tNrv9ttvjzvuuMNgAQC6PU/9BQAAADhE5eTkxHnnnRfnnXdetzqujIyMmD59eqdPxwkAcDgxPSYAAACQUhk5adFneKZCAADQowntAAAAgJTIyEmLMWcfGWfcOST6jcxWEAAAevb3YyUAAAAAulJGTlocdfoRMerTR0Rmb39PDAAAEUI7AAAA6HA5R6bHp787uNse32/u+zCqttZ2+X6FdQAA0ML3ZSUAAACAjpWWSIve+d33v9yJ9LQu3Z+wDgAA2vC9WQkAAACAziCsAwCAdnx/VgIAAACgIwnrAADgAL5HKwEAAADQEYR1AABwEN+nlQAAAAA4GMI6AADogO/VSgAAAAAcCGEdAAB04PdrJQAAAADaQ1gHAACd8D1bCQAAAIC2ENYBAEAnft9WAgAAAKAlwjoAAOiC791KAAAAADRFWAcAAF34/VsJAAAAgH0J6wAAIAXfw5UAAAAAiBDWAQBASr+PKwEAAAD0bMI6AADoBt/LlQAAAAB6JmEdAAB0o+/nSgAAAAA9S0ZOIsacfaSwDgAAutP3dCUAAACAnmXatwZGemaaQgAAQDfiz+kAAACghxHYAQBA9yO0AwAAAAAAgBQT2gEAAAAAAECKeaYdAAAAcMjYsmVLzJ07VyGIyspKRQAADitCOwAAAOCQUVZWFrfeeqtCAABw2DE9JgAAAAAAAKSY0A4AAAAAAABSTGgHAAAAAAAAKSa0AwAAAAAAgBQT2gEAAAAAAECKCe0AAAAAAAAgxYR2AAAAAAAAkGJCOwAAAAAAAEixDCUAAAAAUq1Xr15x5JFHKgQdLicnRxEAgENCWn19fb0yAAAAAAAAQOqYHhMAAAAAAABSTGgHAAAAAAAAKSa0AwAAAAAAgBQT2gEAAAAAAECKCe0AAAAAAAAgxYR2AAAAAAAAkGJCOwAAAAAAAEgxoR0AAAAAAACkmNAOAAAAAAAAUkxoBwAAAAAAACkmtAMAAAAAAIAUE9oBAAAAAABAiv1/i1bIhUTmS3UAAAAASUVORK5CYII=" - } - }, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "![reuse.png](attachment:reuse.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "So now let's make a new configuration for this model, and set the `ReuseFactor` to `2` for every layer:\n", - "we'll compile the model, then evaulate its performance. (Note, by creating a new config with `granularity=Model`, we're implicitly resetting the precision to `ap_fixed<16,6>` throughout.) Changing the `ReuseFactor` should not change the classification results, but let's just verify that by inspecting the accuracy and ROC curve again!\n", - "Then we'll build the model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "config = hls4ml.utils.config_from_keras_model(model, granularity='Model', backend='Vitis')\n", - "print(\"-----------------------------------\")\n", - "print(config)\n", - "print(\"-----------------------------------\")\n", - "# Set the ReuseFactor to 2 throughout\n", - "config['Model']['ReuseFactor'] = 2\n", - "hls_model = hls4ml.converters.convert_from_keras_model(\n", - " model, hls_config=config, backend='vitis', output_dir='model_1/hls4ml_prj_2', part='xcu250-figd2104-2L-e'\n", - ")\n", - "hls_model.compile()\n", - "y_hls = hls_model.predict(X_test)\n", - "print(\"Keras Accuracy: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_keras, axis=1))))\n", - "print(\"hls4ml Accuracy: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_hls, axis=1))))\n", - "plt.figure(figsize=(9, 9))\n", - "_ = plotting.makeRoc(y_test, y_keras, classes)\n", - "plt.gca().set_prop_cycle(None) # reset the colors\n", - "_ = plotting.makeRoc(y_test, y_hls, classes, linestyle='--')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now build the model\n", - "\n", - "**This can take several minutes.**\n", - "\n", - "While the C-Synthesis is running, we can monitor the progress looking at the log file by opening a terminal from the notebook home, and executing:\n", - "\n", - "`tail -f model_1/hls4ml_prj_2/vitis_hls.log`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls_model.build(csim=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And now print the report, compare this to the report from Exercise 1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls4ml.report.read_vivado_report('model_1/hls4ml_prj_2')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls4ml.report.read_vivado_report('model_1/hls4ml_prj')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Exercise\n", - "- Recall the outcome of the exercise of part 1 where we estimated how many DSPs our network should use.\n", - "How does this change now we've used `ReuseFactor = 2` for the network? Does the expectation match the report this time?" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/part3_compression.ipynb b/part3_compression.ipynb deleted file mode 100644 index 97f38088..00000000 --- a/part3_compression.ipynb +++ /dev/null @@ -1,320 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Part 3: Compression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.keras.utils import to_categorical\n", - "from sklearn.datasets import fetch_openml\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.preprocessing import LabelEncoder, StandardScaler\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "%matplotlib inline\n", - "seed = 0\n", - "np.random.seed(seed)\n", - "import tensorflow as tf\n", - "\n", - "tf.random.set_seed(seed)\n", - "import os\n", - "\n", - "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Fetch the jet tagging dataset from Open ML" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "X_train_val = np.load('X_train_val.npy')\n", - "X_test = np.load('X_test.npy')\n", - "y_train_val = np.load('y_train_val.npy')\n", - "y_test = np.load('y_test.npy')\n", - "classes = np.load('classes.npy', allow_pickle=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Now construct a model\n", - "We'll use the same architecture as in part 1: 3 hidden layers with 64, then 32, then 32 neurons. Each layer will use `relu` activation.\n", - "Add an output layer with 5 neurons (one for each class), then finish with Softmax activation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.keras.models import Sequential\n", - "from tensorflow.keras.layers import Dense, Activation, BatchNormalization\n", - "from tensorflow.keras.optimizers import Adam\n", - "from tensorflow.keras.regularizers import l1\n", - "from callbacks import all_callbacks" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model = Sequential()\n", - "model.add(Dense(64, input_shape=(16,), name='fc1', kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001)))\n", - "model.add(Activation(activation='relu', name='relu1'))\n", - "model.add(Dense(32, name='fc2', kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001)))\n", - "model.add(Activation(activation='relu', name='relu2'))\n", - "model.add(Dense(32, name='fc3', kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001)))\n", - "model.add(Activation(activation='relu', name='relu3'))\n", - "model.add(Dense(5, name='output', kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001)))\n", - "model.add(Activation(activation='softmax', name='softmax'))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train sparse\n", - "This time we'll use the Tensorflow model optimization sparsity to train a sparse model (forcing many weights to '0'). In this instance, the target sparsity is 75%" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow_model_optimization.python.core.sparsity.keras import prune, pruning_callbacks, pruning_schedule\n", - "from tensorflow_model_optimization.sparsity.keras import strip_pruning\n", - "\n", - "pruning_params = {\"pruning_schedule\": pruning_schedule.ConstantSparsity(0.75, begin_step=2000, frequency=100)}\n", - "model = prune.prune_low_magnitude(model, **pruning_params)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train the model\n", - "We'll use the same settings as the model for part 1: Adam optimizer with categorical crossentropy loss.\n", - "The callbacks will decay the learning rate and save the model into a directory 'model_2'\n", - "The model isn't very complex, so this should just take a few minutes even on the CPU.\n", - "If you've restarted the notebook kernel after training once, set `train = False` to load the trained model rather than training again." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train = True\n", - "if train:\n", - " adam = Adam(lr=0.0001)\n", - " model.compile(optimizer=adam, loss=['categorical_crossentropy'], metrics=['accuracy'])\n", - " callbacks = all_callbacks(\n", - " stop_patience=1000,\n", - " lr_factor=0.5,\n", - " lr_patience=10,\n", - " lr_epsilon=0.000001,\n", - " lr_cooldown=2,\n", - " lr_minimum=0.0000001,\n", - " outputDir='model_2',\n", - " )\n", - " callbacks.callbacks.append(pruning_callbacks.UpdatePruningStep())\n", - " model.fit(\n", - " X_train_val,\n", - " y_train_val,\n", - " batch_size=1024,\n", - " epochs=10,\n", - " validation_split=0.25,\n", - " shuffle=True,\n", - " callbacks=callbacks.callbacks,\n", - " )\n", - " # Save the model again but with the pruning 'stripped' to use the regular layer types\n", - " model = strip_pruning(model)\n", - " model.save('model_2/KERAS_check_best_model.h5')\n", - "else:\n", - " from tensorflow.keras.models import load_model\n", - "\n", - " model = load_model('model_2/KERAS_check_best_model.h5')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Check sparsity\n", - "Make a quick check that the model was indeed trained sparse. We'll just make a histogram of the weights of the 1st layer, and hopefully observe a large peak in the bin containing '0'. Note logarithmic y axis." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "w = model.layers[0].weights[0].numpy()\n", - "h, b = np.histogram(w, bins=100)\n", - "plt.figure(figsize=(7, 7))\n", - "plt.bar(b[:-1], h, width=b[1] - b[0])\n", - "plt.semilogy()\n", - "print('% of zeros = {}'.format(np.sum(w == 0) / np.size(w)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Check performance\n", - "How does this 75% sparse model compare against the unpruned model? Let's report the accuracy and make a ROC curve. The pruned model is shown with solid lines, the unpruned model from part 1 is shown with dashed lines.\n", - "**Make sure you've trained the model from part 1**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import plotting\n", - "import matplotlib.pyplot as plt\n", - "from sklearn.metrics import accuracy_score\n", - "from tensorflow.keras.models import load_model\n", - "\n", - "model_ref = load_model('model_1/KERAS_check_best_model.h5')\n", - "\n", - "y_ref = model_ref.predict(X_test)\n", - "y_prune = model.predict(X_test)\n", - "\n", - "print(\"Accuracy unpruned: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_ref, axis=1))))\n", - "print(\"Accuracy pruned: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_prune, axis=1))))\n", - "\n", - "fig, ax = plt.subplots(figsize=(9, 9))\n", - "_ = plotting.makeRoc(y_test, y_ref, classes)\n", - "plt.gca().set_prop_cycle(None) # reset the colors\n", - "_ = plotting.makeRoc(y_test, y_prune, classes, linestyle='--')\n", - "\n", - "from matplotlib.lines import Line2D\n", - "\n", - "lines = [Line2D([0], [0], ls='-'), Line2D([0], [0], ls='--')]\n", - "from matplotlib.legend import Legend\n", - "\n", - "leg = Legend(ax, lines, labels=['unpruned', 'pruned'], loc='lower right', frameon=False)\n", - "ax.add_artist(leg)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Convert the model to FPGA firmware with hls4ml\n", - "Let's use the default configuration: `ap_fixed<16,6>` precision everywhere and `ReuseFactor=1`, so we can compare with the part 1 model. We need to use `strip_pruning` to change the layer types back to their originals.\n", - "\n", - "**The synthesis will take a while**\n", - "\n", - "While the C-Synthesis is running, we can monitor the progress looking at the log file by opening a terminal from the notebook home, and executing:\n", - "\n", - "`tail -f model_2/hls4ml_prj/vitis_hls.log`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import hls4ml\n", - "\n", - "config = hls4ml.utils.config_from_keras_model(model, granularity='model', backend='Vitis')\n", - "print(config)\n", - "hls_model = hls4ml.converters.convert_from_keras_model(\n", - " model, hls_config=config, backend='Vitis', output_dir='model_2/hls4ml_prj', part='xcu250-figd2104-2L-e'\n", - ")\n", - "hls_model.compile()\n", - "hls_model.build(csim=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Check the reports\n", - "Print out the reports generated by Vitis HLS. Pay attention to the Utilization Estimates' section in particular this time." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls4ml.report.read_vivado_report('model_2/hls4ml_prj/')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Print the report for the model trained in part 1. Remember these models have the same architecture, but the model in this section was trained using the sparsity API from tensorflow_model_optimization. Notice how the resource usage had dramatically reduced (particularly the DSPs). When Vitis HLS notices an operation like `y = 0 * x` it can avoid placing a DSP for that operation. The impact of this is biggest when `ReuseFactor = 1`, but still applies at higher reuse as well. **Note you need to have trained and synthesized the model from part 1**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls4ml.report.read_vivado_report('model_1/hls4ml_prj')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/part4_quantization.ipynb b/part4_quantization.ipynb deleted file mode 100644 index 830ca203..00000000 --- a/part4_quantization.ipynb +++ /dev/null @@ -1,405 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Part 4: Quantization" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.keras.utils import to_categorical\n", - "from sklearn.datasets import fetch_openml\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.preprocessing import LabelEncoder, StandardScaler\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "%matplotlib inline\n", - "seed = 0\n", - "np.random.seed(seed)\n", - "import tensorflow as tf\n", - "\n", - "tf.random.set_seed(seed)\n", - "import os\n", - "\n", - "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Fetch the jet tagging dataset from Open ML" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "X_train_val = np.load('X_train_val.npy')\n", - "X_test = np.load('X_test.npy')\n", - "y_train_val = np.load('y_train_val.npy')\n", - "y_test = np.load('y_test.npy')\n", - "classes = np.load('classes.npy', allow_pickle=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Construct a model\n", - "This time we're going to use QKeras layers.\n", - "QKeras is \"Quantized Keras\" for deep heterogeneous quantization of ML models.\n", - "\n", - "https://github.com/google/qkeras\n", - "\n", - "It is maintained by Google and we recently added support for QKeras model to hls4ml." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.keras.models import Sequential\n", - "from tensorflow.keras.optimizers import Adam\n", - "from tensorflow.keras.regularizers import l1\n", - "from callbacks import all_callbacks\n", - "from tensorflow.keras.layers import Activation\n", - "from qkeras.qlayers import QDense, QActivation\n", - "from qkeras.quantizers import quantized_bits, quantized_relu" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We're using `QDense` layer instead of `Dense`, and `QActivation` instead of `Activation`. We're also specifying `kernel_quantizer = quantized_bits(6,0,0)`. This will use 6-bits (of which 0 are integer) for the weights. We also use the same quantization for the biases, and `quantized_relu(6)` for 6-bit ReLU activations." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model = Sequential()\n", - "model.add(\n", - " QDense(\n", - " 64,\n", - " input_shape=(16,),\n", - " name='fc1',\n", - " kernel_quantizer=quantized_bits(6, 0, alpha=1),\n", - " bias_quantizer=quantized_bits(6, 0, alpha=1),\n", - " kernel_initializer='lecun_uniform',\n", - " kernel_regularizer=l1(0.0001),\n", - " )\n", - ")\n", - "model.add(QActivation(activation=quantized_relu(6), name='relu1'))\n", - "model.add(\n", - " QDense(\n", - " 32,\n", - " name='fc2',\n", - " kernel_quantizer=quantized_bits(6, 0, alpha=1),\n", - " bias_quantizer=quantized_bits(6, 0, alpha=1),\n", - " kernel_initializer='lecun_uniform',\n", - " kernel_regularizer=l1(0.0001),\n", - " )\n", - ")\n", - "model.add(QActivation(activation=quantized_relu(6), name='relu2'))\n", - "model.add(\n", - " QDense(\n", - " 32,\n", - " name='fc3',\n", - " kernel_quantizer=quantized_bits(6, 0, alpha=1),\n", - " bias_quantizer=quantized_bits(6, 0, alpha=1),\n", - " kernel_initializer='lecun_uniform',\n", - " kernel_regularizer=l1(0.0001),\n", - " )\n", - ")\n", - "model.add(QActivation(activation=quantized_relu(6), name='relu3'))\n", - "model.add(\n", - " QDense(\n", - " 5,\n", - " name='output',\n", - " kernel_quantizer=quantized_bits(6, 0, alpha=1),\n", - " bias_quantizer=quantized_bits(6, 0, alpha=1),\n", - " kernel_initializer='lecun_uniform',\n", - " kernel_regularizer=l1(0.0001),\n", - " )\n", - ")\n", - "model.add(Activation(activation='softmax', name='softmax'))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train sparse\n", - "Let's train with model sparsity again, since QKeras layers are prunable." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow_model_optimization.python.core.sparsity.keras import prune, pruning_callbacks, pruning_schedule\n", - "from tensorflow_model_optimization.sparsity.keras import strip_pruning\n", - "\n", - "pruning_params = {\"pruning_schedule\": pruning_schedule.ConstantSparsity(0.75, begin_step=2000, frequency=100)}\n", - "model = prune.prune_low_magnitude(model, **pruning_params)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train the model\n", - "We'll use the same settings as the model for part 1: Adam optimizer with categorical crossentropy loss.\n", - "The callbacks will decay the learning rate and save the model into a directory 'model_2'\n", - "The model isn't very complex, so this should just take a few minutes even on the CPU.\n", - "If you've restarted the notebook kernel after training once, set `train = False` to load the trained model rather than training again." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train = True\n", - "if train:\n", - " adam = Adam(lr=0.0001)\n", - " model.compile(optimizer=adam, loss=['categorical_crossentropy'], metrics=['accuracy'])\n", - " callbacks = all_callbacks(\n", - " stop_patience=1000,\n", - " lr_factor=0.5,\n", - " lr_patience=10,\n", - " lr_epsilon=0.000001,\n", - " lr_cooldown=2,\n", - " lr_minimum=0.0000001,\n", - " outputDir='model_3',\n", - " )\n", - " callbacks.callbacks.append(pruning_callbacks.UpdatePruningStep())\n", - " model.fit(\n", - " X_train_val,\n", - " y_train_val,\n", - " batch_size=1024,\n", - " epochs=30,\n", - " validation_split=0.25,\n", - " shuffle=True,\n", - " callbacks=callbacks.callbacks,\n", - " )\n", - " # Save the model again but with the pruning 'stripped' to use the regular layer types\n", - " model = strip_pruning(model)\n", - " model.save('model_3/KERAS_check_best_model.h5')\n", - "else:\n", - " from tensorflow.keras.models import load_model\n", - " from qkeras.utils import _add_supported_quantized_objects\n", - "\n", - " co = {}\n", - " _add_supported_quantized_objects(co)\n", - " model = load_model('model_3/KERAS_check_best_model.h5', custom_objects=co)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Check performance\n", - "How does this model which was trained using 6-bits, and 75% sparsity model compare against the original model? Let's report the accuracy and make a ROC curve. The quantized, pruned model is shown with solid lines, the unpruned model from part 1 is shown with dashed lines.\n", - "\n", - "\n", - "We should also check that hls4ml can respect the choice to use 6-bits throughout the model, and match the accuracy. We'll generate a configuration from this Quantized model, and plot its performance as the dotted line.\n", - "The generated configuration is printed out. You'll notice that it uses 7 bits for the type, but we specified 6!? That's just because QKeras doesn't count the sign-bit when we specify the number of bits, so the type that actually gets used needs 1 more.\n", - "\n", - "We also use the `OutputRoundingSaturationMode` optimizer pass of `hls4ml` to set the Activation layers to round, rather than truncate, the cast. This is important for getting good model accuracy when using small bit precision activations. And we'll set a different data type for the tables used in the Softmax, just for a bit of extra performance.\n", - "\n", - "\n", - "**Make sure you've trained the model from part 1**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import hls4ml\n", - "import plotting\n", - "\n", - "config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend='Vitis')\n", - "config['LayerName']['softmax']['exp_table_t'] = 'ap_fixed<18,8>'\n", - "config['LayerName']['softmax']['inv_table_t'] = 'ap_fixed<18,4>'\n", - "print(\"-----------------------------------\")\n", - "plotting.print_dict(config)\n", - "print(\"-----------------------------------\")\n", - "hls_model = hls4ml.converters.convert_from_keras_model(\n", - " model, hls_config=config, backend='Vitis', output_dir='model_3/hls4ml_prj', part='xcu250-figd2104-2L-e'\n", - ")\n", - "hls_model.compile()\n", - "\n", - "y_qkeras = model.predict(np.ascontiguousarray(X_test))\n", - "y_hls = hls_model.predict(np.ascontiguousarray(X_test))\n", - "np.save('model_3/y_qkeras.npy', y_qkeras)\n", - "np.save('model_3/y_hls.npy', y_hls)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "from sklearn.metrics import accuracy_score\n", - "from tensorflow.keras.models import load_model\n", - "\n", - "model_ref = load_model('model_1/KERAS_check_best_model.h5')\n", - "y_ref = model_ref.predict(X_test)\n", - "\n", - "print(\"Accuracy baseline: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_ref, axis=1))))\n", - "print(\"Accuracy pruned, quantized: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_qkeras, axis=1))))\n", - "print(\"Accuracy hls4ml: {}\".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_hls, axis=1))))\n", - "\n", - "fig, ax = plt.subplots(figsize=(9, 9))\n", - "_ = plotting.makeRoc(y_test, y_ref, classes)\n", - "plt.gca().set_prop_cycle(None) # reset the colors\n", - "_ = plotting.makeRoc(y_test, y_qkeras, classes, linestyle='--')\n", - "plt.gca().set_prop_cycle(None) # reset the colors\n", - "_ = plotting.makeRoc(y_test, y_hls, classes, linestyle=':')\n", - "\n", - "from matplotlib.lines import Line2D\n", - "\n", - "lines = [Line2D([0], [0], ls='-'), Line2D([0], [0], ls='--'), Line2D([0], [0], ls=':')]\n", - "from matplotlib.legend import Legend\n", - "\n", - "leg = Legend(ax, lines, labels=['baseline', 'pruned, quantized', 'hls4ml'], loc='lower right', frameon=False)\n", - "ax.add_artist(leg)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Synthesize\n", - "Now let's synthesize this quantized, pruned model.\n", - "\n", - "**The synthesis will take a while**\n", - "\n", - "While the C-Synthesis is running, we can monitor the progress looking at the log file by opening a terminal from the notebook home, and executing:\n", - "\n", - "`tail -f model_3/hls4ml_prj/vitis_hls.log`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls_model.build(csim=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Check the reports\n", - "Print out the reports generated by Vitis HLS. Pay attention to the Utilization Estimates' section in particular this time." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls4ml.report.read_vivado_report('model_3/hls4ml_prj')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Print the report for the model trained in part 1. Now, compared to the model from part 1, this model has been trained with low-precision quantization, and 75% pruning. You should be able to see that we have saved a lot of resource compared to where we started in part 1. At the same time, referring to the ROC curve above, the model performance is pretty much identical even with this drastic compression!\n", - "\n", - "**Note you need to have trained and synthesized the model from part 1**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls4ml.report.read_vivado_report('model_1/hls4ml_prj')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Print the report for the model trained in part 3. Both these models were trained with 75% sparsity, but the new model uses 6-bit precision as well. You can see how Vitis HLS has moved multiplication operations from DSPs into LUTs, reducing the \"critical\" resource usage.\n", - "\n", - "**Note you need to have trained and synthesized the model from part 3**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls4ml.report.read_vivado_report('model_2/hls4ml_prj')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## NB\n", - "Note as well that the Vitis HLS resource estimates tend to _overestimate_ LUTs, while generally estimating the DSPs correctly. Running the subsequent stages of FPGA compilation reveals the more realistic resource usage, You can run the next step, 'logic synthesis' with `hls_model.build(synth=True, vsynth=True)`, but we skipped it in this tutorial in the interest of time." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/part6_cnns.ipynb b/part6_cnns.ipynb deleted file mode 100644 index dd940a1a..00000000 --- a/part6_cnns.ipynb +++ /dev/null @@ -1,1326 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "# Part 6: Convolutional Neural Networks in hls4ml\n", - "\n", - "In this notebook you will learn how to train a pruned and quantized convolutional neural network (CNN) and deploy it using hls4ml. For this exercise, we will use the Street View House Numbers (SVHN) Dataset (http://ufldl.stanford.edu/housenumbers/).\n", - "\n", - "The SVHN dataset consists of real-world images of house numbers extracted from Google Street View images. The format is similar to that of the MNIST dataset, but is a much more challenging real-world problem, as illustrated by the examples shown below.\n", - "\n", - "All the images are in RGB format and have been cropped to 32x32 pixels. \n", - "Unlike MNIST, more than one digit can be present in the same image and in these cases, the center digit is used to assign a label to the image.\n", - "Each image can belong to one of 10 classes, corresponding to digits 0 through 9.\n", - "\n", - "![alt text](images/test.png \"SVHN examples from the test dataset\")\n", - "\n", - "The SVHN dataset consists of 73,257 images for training (and 531,131 extra samples that are easier to classify and can be used as additional training data) and 26,032 images for testing." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "## Start with the neccessary imports" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import time\n", - "import tensorflow.compat.v2 as tf\n", - "import tensorflow_datasets as tfds\n", - "\n", - "os.environ['PATH'] = os.environ['XILINX_VITIS'] + '/bin:' + os.environ['PATH']" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "## Fetch the SVHN dataset using Tensorflow Dataset\n", - "\n", - "In this part we will fetch the trainining, validation and test dataset using Tensorflow Datasets (https://www.tensorflow.org/datasets). We will not use the 'extra' training in order to save time, but you could fetch it by adding `split='train[:90%]+extra'`. We will use the first 90% of the training data for training and the last 10% for validation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_train, info = tfds.load('svhn_cropped', split='train[:90%]', with_info=True, as_supervised=True)\n", - "ds_test = tfds.load('svhn_cropped', split='test', shuffle_files=True, as_supervised=True)\n", - "ds_val = tfds.load('svhn_cropped', split='train[-10%:]', shuffle_files=True, as_supervised=True)\n", - "\n", - "assert isinstance(ds_train, tf.data.Dataset)\n", - "train_size = int(info.splits['train'].num_examples)\n", - "input_shape = info.features['image'].shape\n", - "n_classes = info.features['label'].num_classes\n", - "\n", - "print('Training on {} samples of input shape {}, belonging to {} classes'.format(train_size, input_shape, n_classes))\n", - "fig = tfds.show_examples(ds_train, info)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "We'll use TensorFlow Dataset to prepare our datasets. We'll fetch the training dataset as tuples, and the test dataset as numpy arrays" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def preprocess(image, label, nclasses=10):\n", - " image = tf.cast(image, tf.float32) / 255.0\n", - " label = tf.one_hot(tf.squeeze(label), nclasses)\n", - " return image, label" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 1024\n", - "\n", - "train_data = ds_train.map(preprocess, n_classes) # Get dataset as image and one-hot encoded labels, divided by max RGB\n", - "train_data = train_data.shuffle(4096).batch(batch_size).prefetch(tf.data.experimental.AUTOTUNE)\n", - "\n", - "for example in train_data.take(1):\n", - " break\n", - "print(\"X train batch shape = {}, Y train batch shape = {} \".format(example[0].shape, example[1].shape))\n", - "\n", - "val_data = ds_val.map(preprocess, n_classes)\n", - "val_data = val_data.batch(batch_size)\n", - "val_data = val_data.prefetch(tf.data.experimental.AUTOTUNE)\n", - "\n", - "# For testing, we get the full dataset in memory as it's rather small.\n", - "# We fetch it as numpy arrays to have access to labels and images separately\n", - "X_test, Y_test = tfds.as_numpy(tfds.load('svhn_cropped', split='test', batch_size=-1, as_supervised=True))\n", - "X_test, Y_test = preprocess(X_test, Y_test, nclasses=n_classes)\n", - "print(\"X test batch shape = {}, Y test batch shape = {} \".format(X_test.shape, Y_test.shape))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "## Defining the model\n", - "\n", - "We then need to define a model. For the lowest possible latency, each layer should have a maximum number of trainable parameters of 4096. This is due to fixed limits in the Vivado compiler, beyond which maximally unrolled (=parallel) compilation will fail. This will allow us to use `strategy = 'latency'` in the hls4ml part, rather than `strategy = 'resource'`, in turn resulting in lower latency" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.keras.layers import Input\n", - "from tensorflow.keras.layers import BatchNormalization\n", - "from tensorflow.keras.layers import Conv2D\n", - "from tensorflow.keras.regularizers import l1\n", - "from tensorflow.keras.layers import MaxPooling2D\n", - "from tensorflow.keras.layers import Activation\n", - "from tensorflow.keras.layers import Flatten\n", - "from tensorflow.keras.layers import Dense\n", - "\n", - "from tensorflow.keras.models import Model\n", - "\n", - "filters_per_conv_layer = [16, 16, 24]\n", - "neurons_per_dense_layer = [42, 64]\n", - "\n", - "x = x_in = Input(input_shape)\n", - "\n", - "for i, f in enumerate(filters_per_conv_layer):\n", - " print(('Adding convolutional block {} with N={} filters').format(i, f))\n", - " x = Conv2D(\n", - " int(f),\n", - " kernel_size=(3, 3),\n", - " strides=(1, 1),\n", - " kernel_initializer='lecun_uniform',\n", - " kernel_regularizer=l1(0.0001),\n", - " use_bias=False,\n", - " name='conv_{}'.format(i),\n", - " )(x)\n", - " x = BatchNormalization(name='bn_conv_{}'.format(i))(x)\n", - " x = Activation('relu', name='conv_act_%i' % i)(x)\n", - " x = MaxPooling2D(pool_size=(2, 2), name='pool_{}'.format(i))(x)\n", - "x = Flatten()(x)\n", - "\n", - "for i, n in enumerate(neurons_per_dense_layer):\n", - " print(('Adding dense block {} with N={} neurons').format(i, n))\n", - " x = Dense(n, kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001), name='dense_%i' % i, use_bias=False)(x)\n", - " x = BatchNormalization(name='bn_dense_{}'.format(i))(x)\n", - " x = Activation('relu', name='dense_act_%i' % i)(x)\n", - "x = Dense(int(n_classes), name='output_dense')(x)\n", - "x_out = Activation('softmax', name='output_softmax')(x)\n", - "\n", - "model = Model(inputs=[x_in], outputs=[x_out], name='keras_baseline')\n", - "\n", - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "Lets check if this model can be implemented completely unrolled (=parallel)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for layer in model.layers:\n", - " if layer.__class__.__name__ in ['Conv2D', 'Dense']:\n", - " w = layer.get_weights()[0]\n", - " layersize = np.prod(w.shape)\n", - " print(\"{}: {}\".format(layer.name, layersize)) # 0 = weights, 1 = biases\n", - " if layersize > 4096: # assuming that shape[0] is batch, i.e., 'None'\n", - " print(\"Layer {} is too large ({}), are you sure you want to train?\".format(layer.name, layersize))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "Looks good! It's below the Vivado-enforced unroll limit of 4096.\n", - "\n", - "## Prune dense and convolutional layers\n", - "Since we've seen in the previous notebooks that pruning can be done at no accuracy cost, let's prune the convolutional and dense layers to 50% sparsity, skipping the output layer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow_model_optimization as tfmot\n", - "from tensorflow_model_optimization.sparsity import keras as sparsity\n", - "from tensorflow_model_optimization.python.core.sparsity.keras import pruning_callbacks\n", - "\n", - "NSTEPS = int(train_size * 0.9) // batch_size # 90% train, 10% validation in 10-fold cross validation\n", - "print('Number of training steps per epoch is {}'.format(NSTEPS))\n", - "\n", - "\n", - "# Prune all convolutional and dense layers gradually from 0 to 50% sparsity every 2 epochs,\n", - "# ending by the 10th epoch\n", - "def pruneFunction(layer):\n", - " pruning_params = {\n", - " 'pruning_schedule': sparsity.PolynomialDecay(\n", - " initial_sparsity=0.0, final_sparsity=0.50, begin_step=NSTEPS * 2, end_step=NSTEPS * 10, frequency=NSTEPS\n", - " )\n", - " }\n", - " if isinstance(layer, tf.keras.layers.Conv2D):\n", - " return tfmot.sparsity.keras.prune_low_magnitude(layer, **pruning_params)\n", - " if isinstance(layer, tf.keras.layers.Dense) and layer.name != 'output_dense':\n", - " return tfmot.sparsity.keras.prune_low_magnitude(layer, **pruning_params)\n", - " return layer\n", - "\n", - "\n", - "model_pruned = tf.keras.models.clone_model(model, clone_function=pruneFunction)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "## Train baseline\n", - "\n", - "We're now ready to train the model! We defined the batch size and n epochs above. We won't use callbacks that store the best weights only, since this might select a weight configuration that has not yet reached 50% sparsity." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train = True # True if you want to retrain, false if you want to load a previsously trained model\n", - "\n", - "n_epochs = 30\n", - "\n", - "if train:\n", - " LOSS = tf.keras.losses.CategoricalCrossentropy()\n", - " OPTIMIZER = tf.keras.optimizers.Adam(learning_rate=3e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=True)\n", - "\n", - " model_pruned.compile(loss=LOSS, optimizer=OPTIMIZER, metrics=[\"accuracy\"])\n", - "\n", - " callbacks = [\n", - " tf.keras.callbacks.EarlyStopping(patience=10, verbose=1),\n", - " tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, verbose=1),\n", - " pruning_callbacks.UpdatePruningStep(),\n", - " ]\n", - "\n", - " start = time.time()\n", - " model_pruned.fit(train_data, epochs=n_epochs, validation_data=val_data, callbacks=callbacks)\n", - " end = time.time()\n", - "\n", - " print('It took {} minutes to train Keras model'.format((end - start) / 60.0))\n", - "\n", - " model_pruned.save('pruned_cnn_model.h5')\n", - "\n", - "else:\n", - " from qkeras.utils import _add_supported_quantized_objects\n", - " from tensorflow_model_optimization.python.core.sparsity.keras import pruning_wrapper\n", - "\n", - " co = {}\n", - " _add_supported_quantized_objects(co)\n", - " co['PruneLowMagnitude'] = pruning_wrapper.PruneLowMagnitude\n", - " model_pruned = tf.keras.models.load_model('pruned_cnn_model.h5', custom_objects=co)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "You'll notice the accuracy is lower than that in the hls4ml CNN paper (https://arxiv.org/abs/2101.05108) despite the model being the same. The reson for this is that we didn't use the ``extra`` training data in order to save time. If you want to futher optimize the network, increasing the training data is a good place to start. Enlarging the model architecture comes at a high latency/resource cost." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "## Quantization and the fused Conv2D+BatchNormalization layer in QKeras\n", - "Let's now create a pruned an quantized model using QKeras. For this, we will use a fused Convolutional and BatchNormalization (BN) layer from QKeras, which will further speed up the implementation when we implement the model using hls4ml. \n", - "There is currently no fused Dense+BatchNoralization layer available in QKeras, so we'll use Keras BatchNormalization when BN follows a Dense layer for now. We'll use the same precision everywhere, namely a bit width of 6 and 0 integer bits (this will be implemented as``<6,1>`` in hls4ml, due to the missing sign-bit). For now, make sure to set ```use_bias=True``` in ```QConv2DBatchnorm``` to avoid problems during synthesis." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from qkeras import QActivation\n", - "from qkeras import QDense, QConv2DBatchnorm\n", - "\n", - "x = x_in = Input(shape=input_shape)\n", - "\n", - "for i, f in enumerate(filters_per_conv_layer):\n", - " print(('Adding fused QConv+BN block {} with N={} filters').format(i, f))\n", - " x = QConv2DBatchnorm(\n", - " int(f),\n", - " kernel_size=(3, 3),\n", - " strides=(1, 1),\n", - " kernel_quantizer=\"quantized_bits(6,0,alpha=1)\",\n", - " bias_quantizer=\"quantized_bits(6,0,alpha=1)\",\n", - " kernel_initializer='lecun_uniform',\n", - " kernel_regularizer=l1(0.0001),\n", - " use_bias=True,\n", - " name='fused_convbn_{}'.format(i),\n", - " )(x)\n", - " x = QActivation('quantized_relu(6)', name='conv_act_%i' % i)(x)\n", - " x = MaxPooling2D(pool_size=(2, 2), name='pool_{}'.format(i))(x)\n", - "x = Flatten()(x)\n", - "\n", - "for i, n in enumerate(neurons_per_dense_layer):\n", - " print(('Adding QDense block {} with N={} neurons').format(i, n))\n", - " x = QDense(\n", - " n,\n", - " kernel_quantizer=\"quantized_bits(6,0,alpha=1)\",\n", - " kernel_initializer='lecun_uniform',\n", - " kernel_regularizer=l1(0.0001),\n", - " name='dense_%i' % i,\n", - " use_bias=False,\n", - " )(x)\n", - " x = BatchNormalization(name='bn_dense_{}'.format(i))(x)\n", - " x = QActivation('quantized_relu(6)', name='dense_act_%i' % i)(x)\n", - "x = Dense(int(n_classes), name='output_dense')(x)\n", - "x_out = Activation('softmax', name='output_softmax')(x)\n", - "qmodel = Model(inputs=[x_in], outputs=[x_out], name='qkeras')\n", - "\n", - "qmodel.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Print the quantized layers\n", - "from qkeras.autoqkeras.utils import print_qmodel_summary\n", - "\n", - "print_qmodel_summary(qmodel)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "You see that a bias quantizer is defined, although we are not using a bias term for the layers. This is set automatically by QKeras. In addition, you'll note that ``alpha='1'``. This sets the weight scale per channel to 1 (no scaling). The default is ``alpha='auto_po2'``, which sets the weight scale per channel to be a power-of-2, such that an actual hardware implementation can be performed by just shifting the result of the convolutional/dense layer to the right or left by checking the sign of the scale and then taking the log2 of the scale.\n", - "\n", - "Let's now prune and train this model! If you want, you can also train the unpruned version, ``qmodel`` and see how the performance compares. We will stick to the pruned one here. Again, we do not use a model checkpoint which stores the best weights, in order to ensure the model is trained to the desired sparsity." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "qmodel_pruned = tf.keras.models.clone_model(qmodel, clone_function=pruneFunction)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train = True\n", - "\n", - "n_epochs = 30\n", - "if train:\n", - " LOSS = tf.keras.losses.CategoricalCrossentropy()\n", - " OPTIMIZER = tf.keras.optimizers.Adam(learning_rate=3e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=True)\n", - " qmodel_pruned.compile(loss=LOSS, optimizer=OPTIMIZER, metrics=[\"accuracy\"])\n", - "\n", - " callbacks = [\n", - " tf.keras.callbacks.EarlyStopping(patience=10, verbose=1),\n", - " tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, verbose=1),\n", - " pruning_callbacks.UpdatePruningStep(),\n", - " ]\n", - "\n", - " start = time.time()\n", - " history = qmodel_pruned.fit(train_data, epochs=n_epochs, validation_data=val_data, callbacks=callbacks, verbose=1)\n", - " end = time.time()\n", - " print('\\n It took {} minutes to train!\\n'.format((end - start) / 60.0))\n", - "\n", - " qmodel_pruned.save('quantized_pruned_cnn_model.h5')\n", - "\n", - "else:\n", - " from qkeras.utils import _add_supported_quantized_objects\n", - " from tensorflow_model_optimization.python.core.sparsity.keras import pruning_wrapper\n", - "\n", - " co = {}\n", - " _add_supported_quantized_objects(co)\n", - " co['PruneLowMagnitude'] = pruning_wrapper.PruneLowMagnitude\n", - " qmodel_pruned = tf.keras.models.load_model('quantized_pruned_cnn_model.h5', custom_objects=co)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "We note that training a model quantization aware, takes around twice as long as when not quantizing during training!\n", - "The validation accuracy is very similar to that of the floating point model equivalent, despite containing significantly less information \n", - "\n", - "## Performance\n", - "Let's look at some ROC curves to compare the performance. Lets choose a few numbers so it doesn't get confusing. Feel free to change the numbers in ``labels``." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "predict_baseline = model_pruned.predict(X_test)\n", - "test_score_baseline = model_pruned.evaluate(X_test, Y_test)\n", - "\n", - "predict_qkeras = qmodel_pruned.predict(X_test)\n", - "test_score_qkeras = qmodel_pruned.evaluate(X_test, Y_test)\n", - "\n", - "print('Keras accuracy = {} , QKeras 6-bit accuracy = {}'.format(test_score_baseline[1], test_score_qkeras[1]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "from sklearn import metrics\n", - "\n", - "labels = ['%i' % nr for nr in range(0, n_classes)] # If you want to look at all the labels\n", - "# labels = ['0','1','9'] # Look at only a few labels, here for digits 0, 1 and 9\n", - "print('Plotting ROC for labels {}'.format(labels))\n", - "\n", - "df = pd.DataFrame()\n", - "df_q = pd.DataFrame()\n", - "fpr = {}\n", - "tpr = {}\n", - "auc1 = {}\n", - "fpr_q = {}\n", - "tpr_q = {}\n", - "auc1_q = {}\n", - "%matplotlib inline\n", - "colors = ['#67001f', '#b2182b', '#d6604d', '#f4a582', '#fddbc7', '#d1e5f0', '#92c5de', '#4393c3', '#2166ac', '#053061']\n", - "fig, ax = plt.subplots(figsize=(10, 10))\n", - "for i, label in enumerate(labels):\n", - " df[label] = Y_test[:, int(label)]\n", - " df[label + '_pred'] = predict_baseline[:, int(label)]\n", - " fpr[label], tpr[label], threshold = metrics.roc_curve(df[label], df[label + '_pred'])\n", - " auc1[label] = metrics.auc(fpr[label], tpr[label])\n", - "\n", - " df_q[label] = Y_test[:, int(label)]\n", - " df_q[label + '_pred'] = predict_qkeras[:, int(label)]\n", - " fpr_q[label], tpr_q[label], threshold_q = metrics.roc_curve(df_q[label], df_q[label + '_pred'])\n", - " auc1_q[label] = metrics.auc(fpr_q[label], tpr_q[label])\n", - "\n", - " plt.plot(\n", - " fpr[label],\n", - " tpr[label],\n", - " label=r'{}, AUC Keras = {:.1f}% AUC QKeras = {:.1f}%)'.format(label, auc1[label] * 100, auc1_q[label] * 100),\n", - " linewidth=1.5,\n", - " c=colors[i],\n", - " linestyle='solid',\n", - " )\n", - " plt.plot(fpr_q[label], tpr_q[label], linewidth=1.5, c=colors[i], linestyle='dotted')\n", - "\n", - "plt.semilogx()\n", - "plt.ylabel(\"True Positive Rate\")\n", - "plt.xlabel(\"False Positive Rate\")\n", - "plt.xlim(0.01, 1.0)\n", - "plt.ylim(0.5, 1.1)\n", - "plt.legend(loc='lower right')\n", - "plt.figtext(\n", - " 0.2,\n", - " 0.83,\n", - " r'Accuracy Keras = {:.1f}% QKeras 8-bit = {:.1f}%'.format(test_score_baseline[1] * 100, test_score_qkeras[1] * 100),\n", - " wrap=True,\n", - " horizontalalignment='left',\n", - " verticalalignment='center',\n", - ")\n", - "from matplotlib.lines import Line2D\n", - "\n", - "lines = [Line2D([0], [0], ls='-'), Line2D([0], [0], ls='--')]\n", - "from matplotlib.legend import Legend\n", - "\n", - "leg = Legend(ax, lines, labels=['Keras', 'QKeras'], loc='lower right', frameon=False)\n", - "ax.add_artist(leg)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "The difference in AUC between the fp32 Keras model and the 8-bit QKeras model, is small, as we have seen for the previous examples. You can find a bonus exercise below, **Bonus: Automatic quantization**, where we'll use AutoQKeras to find the best heterogeneously quantized model, given a set of resource and accuracy constriants.\n", - "### Check sparsity\n", - "Let's also check the per-layer sparsity:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def doWeights(model):\n", - " allWeightsByLayer = {}\n", - " for layer in model.layers:\n", - " if (layer._name).find(\"batch\") != -1 or len(layer.get_weights()) < 1:\n", - " continue\n", - " weights = layer.weights[0].numpy().flatten()\n", - " allWeightsByLayer[layer._name] = weights\n", - " print('Layer {}: % of zeros = {}'.format(layer._name, np.sum(weights == 0) / np.size(weights)))\n", - "\n", - " labelsW = []\n", - " histosW = []\n", - "\n", - " for key in reversed(sorted(allWeightsByLayer.keys())):\n", - " labelsW.append(key)\n", - " histosW.append(allWeightsByLayer[key])\n", - "\n", - " fig = plt.figure(figsize=(10, 10))\n", - " bins = np.linspace(-1.5, 1.5, 50)\n", - " plt.hist(histosW, bins, histtype='stepfilled', stacked=True, label=labelsW, edgecolor='black')\n", - " plt.legend(frameon=False, loc='upper left')\n", - " plt.ylabel('Number of Weights')\n", - " plt.xlabel('Weights')\n", - " plt.figtext(0.2, 0.38, model._name, wrap=True, horizontalalignment='left', verticalalignment='center')\n", - "\n", - "\n", - "doWeights(model_pruned)\n", - "doWeights(qmodel_pruned)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "We see that 50% of the weights per layer are set to zero, as expected.\n", - "Now, let's synthesize the floating point Keras model and the QKeras quantized model!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "## CNNs in hls4ml\n", - "\n", - "In this part, we will take the two models we trained above (the floating-point 32 Keras model and the 6-bit QKeras model), and synthesize them with hls4ml. Although your models are probably already in memory, let's load them from scratch. We need to pass the appropriate custom QKeras/pruning layers when loading, and remove the pruning parameters that were saved together with the model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow_model_optimization.sparsity.keras import strip_pruning\n", - "from tensorflow_model_optimization.python.core.sparsity.keras import pruning_wrapper\n", - "\n", - "from qkeras.utils import _add_supported_quantized_objects\n", - "\n", - "co = {}\n", - "_add_supported_quantized_objects(co)\n", - "co['PruneLowMagnitude'] = pruning_wrapper.PruneLowMagnitude\n", - "\n", - "model = tf.keras.models.load_model('pruned_cnn_model.h5', custom_objects=co)\n", - "model = strip_pruning(model)\n", - "\n", - "qmodel = tf.keras.models.load_model('quantized_pruned_cnn_model.h5', custom_objects=co)\n", - "qmodel = strip_pruning(qmodel)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, we need to define the hls4ml and Vivado configurations. Two things will change with respect to what was done in the previous exercises. First, we will use ``io_type='io_stream'`` in the Vitis_HLS configuration.\n", - "\n", - "---\n", - "****You must use ``io_type='io_stream'`` if attempting to synthesize a large convolutional neural network.****\n", - "\n", - "---\n", - "The CNN implementation in hls4ml is based on streams, which are synthesized in hardware as first in, first out (FIFO) buffers. Shift registers are used to keep track of the last ```` rows of input pixels, and maintains a shifting snapshot of the convolution kernel.\n", - "\n", - "This is illustrated in the gif below. Here, the input image is at the top-left and the output image at the bottom left. The top right image shows the internal state of the shift registers and convolutional kernel. The red square indicates the current pixels contained within the convolutional kernel.\n", - "\n", - "![alt text](images/conv2d_animation.gif \"The implementation of convolutional layers in hls4ml.\")\n", - "\n", - "Lastly, we will use ``['Strategy'] = 'Latency'`` for all the layers in the hls4ml configuration. If one layer would have >4096 elements, we sould set ``['Strategy'] = 'Resource'`` for that layer, or increase the reuse factor by hand. You can find examples of how to do this below.\n", - "\n", - "**NOTE** Using `auto` precision can lead to undesired side effects. In case of this model, the bit width used for the output of the last fully connected layer is larger than can be reasonably represented with the look-up table in the softmax implementation. We therefore need to restrict it by hand to achieve proper results.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import hls4ml\n", - "import plotting\n", - "\n", - "# First, the baseline model\n", - "hls_config = hls4ml.utils.config_from_keras_model(\n", - " model, granularity='name', backend='Vitis', default_precision='ap_fixed<16,6>'\n", - ")\n", - "hls_config['LayerName']['output_dense']['Precision']['result'] = 'fixed<16,6,RND,SAT>'\n", - "plotting.print_dict(hls_config)\n", - "\n", - "\n", - "hls_model = hls4ml.converters.convert_from_keras_model(\n", - " model,\n", - " hls_config=hls_config,\n", - " backend='Vitis',\n", - " output_dir='model_1/hls4ml_prj',\n", - " part='xcu250-figd2104-2L-e',\n", - " io_type='io_stream',\n", - ")\n", - "hls_model.compile()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "Let's get a nice overview over the various shapes and precisions used for each layer through ``hls4ml.utils.plot_model``, as well as look at the weight profile using ``hls4ml.model.profiling.numerical``. The weight profiling returns two plots: Before (top) and after (bottom) various optimizations applied to the HLS model before the final translation to HLS, for instance the fusing of Dense and BatchNormalization layers." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls4ml.utils.plot_model(hls_model, show_shapes=True, show_precision=True, to_file=None)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from hls4ml.model.profiling import numerical\n", - "\n", - "numerical(model=model, hls_model=hls_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The colored boxes are the distribution of the weights of the model, and the gray band illustrates the numerical range covered by the chosen fixed point precision. As we configured, this model uses a precision of ``ap_fixed<16,6>`` for the weights and biases of all layers of the model. \n", - "\n", - "Let's now build our QKeras model. \n", - "\n", - "**NOTE** Using `auto` precision can lead to undesired side effects. In case of this model, the bit width used for the output of the last fully connected layer is larger than can be reasonably represented with the look-up table in the softmax implementation. We therefore need to restrict it by hand to achieve proper results." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Then the QKeras model\n", - "hls_config_q = hls4ml.utils.config_from_keras_model(qmodel, granularity='name', backend='Vitis')\n", - "hls_config_q['LayerName']['output_dense']['Precision']['result'] = 'fixed<16,6,RND,SAT>'\n", - "\n", - "plotting.print_dict(hls_config_q)\n", - "\n", - "hls_model_q = hls4ml.converters.convert_from_keras_model(\n", - " qmodel, hls_config=hls_config_q, output_dir='quantized_pruned_cnn', backend='Vitis', io_type='io_stream'\n", - ")\n", - "\n", - "hls_model_q.compile()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "Let's plot the model and profile the weights her too" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "numerical(model=qmodel, hls_model=hls_model_q)\n", - "hls4ml.utils.plot_model(hls_model_q, show_shapes=True, show_precision=True, to_file=None)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "For the 6-bit QKeras model, we see that different precisions are used for different layers." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "### Accuracy with bit-accurate emulation \n", - "Let's check that the hls4ml accuracy matches the original. This usually takes some time, so let's do it over a reduced dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "X_test_reduced = X_test[:3000]\n", - "Y_test_reduced = Y_test[:3000]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "y_predict = model.predict(X_test_reduced)\n", - "y_predict_hls4ml = hls_model.predict(np.ascontiguousarray(X_test_reduced))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "y_predict_q = qmodel.predict(X_test_reduced)\n", - "y_predict_hls4ml_q = hls_model_q.predict(np.ascontiguousarray(X_test_reduced))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import plotting\n", - "from sklearn.metrics import accuracy_score\n", - "\n", - "\n", - "def plotROC(Y, y_pred, y_pred_hls4ml, label=\"Model\"):\n", - " accuracy_keras = float(accuracy_score(np.argmax(Y, axis=1), np.argmax(y_pred, axis=1)))\n", - " accuracy_hls4ml = float(accuracy_score(np.argmax(Y, axis=1), np.argmax(y_pred_hls4ml, axis=1)))\n", - "\n", - " print(\"Accuracy Keras: {}\".format(accuracy_keras))\n", - " print(\"Accuracy hls4ml: {}\".format(accuracy_hls4ml))\n", - "\n", - " fig, ax = plt.subplots(figsize=(9, 9))\n", - " _ = plotting.makeRoc(Y, y_pred, labels=['%i' % nr for nr in range(n_classes)])\n", - " plt.gca().set_prop_cycle(None) # reset the colors\n", - " _ = plotting.makeRoc(Y, y_pred_hls4ml, labels=['%i' % nr for nr in range(n_classes)], linestyle='--')\n", - "\n", - " from matplotlib.lines import Line2D\n", - "\n", - " lines = [Line2D([0], [0], ls='-'), Line2D([0], [0], ls='--')]\n", - " from matplotlib.legend import Legend\n", - "\n", - " leg = Legend(ax, lines, labels=['Keras', 'hls4ml'], loc='lower right', frameon=False)\n", - " ax.add_artist(leg)\n", - " plt.figtext(0.2, 0.38, label, wrap=True, horizontalalignment='left', verticalalignment='center')\n", - " plt.ylim(0.01, 1.0)\n", - " plt.xlim(0.7, 1.0)\n", - "\n", - "\n", - "# Plot the pruned floating point model:\n", - "plotROC(Y_test_reduced, y_predict, y_predict_hls4ml, label=\"Keras\")\n", - "\n", - "# Plot the pruned and quantized QKeras model\n", - "plotROC(Y_test_reduced, y_predict_q, y_predict_hls4ml_q, label=\"QKeras\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "Looks good! Let's synthesize the models. \n", - "## Logic synthesis\n", - "This takes quite a while for CNN models, up to one hour for the models considered here. In the interest of time, we have therefore provided the neccessary reports for the models considered. You can also synthesize them yourself if you have time, and as usual follow the progress using ``tail -f pruned_cnn/vivado_hls.log`` and ``tail -f quantized_pruned_cnn/vivado_hls.log``.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "synth = False # Only if you want to synthesize the models yourself (>1h per model) rather than look at the provided reports.\n", - "if synth:\n", - " hls_model.build(csim=False, synth=True, vsynth=True)\n", - " hls_model_q.build(csim=False, synth=True, vsynth=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "We extract the latency from the C synthesis, namely the report in ```/myproject_prj/solution1/syn/report/myproject_csynth.rpt```. A more accurate latency estimate can be obtained from running cosim by passing ```hls_model.build(csim=False, synth=True, vsynth=True, cosim=True)``` ( = C/RTL cosimulation, synthesised HLS code is run on a simulator and tested on C test bench) but this takes a lot of time so we will skip it here.\n", - "The resource estimates are obtained from the Vivado logic synthesis, and can be extracted from the report in ```/vivado_synth.rpt```. Let's fetch the most relevant numbers:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def getReports(indir):\n", - " data_ = {}\n", - "\n", - " report_vsynth = Path('{}/vivado_synth.rpt'.format(indir))\n", - " report_csynth = Path('{}/myproject_prj/solution1/syn/report/myproject_csynth.rpt'.format(indir))\n", - "\n", - " if report_vsynth.is_file() and report_csynth.is_file():\n", - " print('Found valid vsynth and synth in {}! Fetching numbers'.format(indir))\n", - "\n", - " # Get the resources from the logic synthesis report\n", - " with report_vsynth.open() as report:\n", - " lines = np.array(report.readlines())\n", - " data_['lut'] = int(lines[np.array(['CLB LUTs*' in line for line in lines])][0].split('|')[2])\n", - " data_['ff'] = int(lines[np.array(['CLB Registers' in line for line in lines])][0].split('|')[2])\n", - " data_['bram'] = float(lines[np.array(['Block RAM Tile' in line for line in lines])][0].split('|')[2])\n", - " data_['dsp'] = int(lines[np.array(['DSPs' in line for line in lines])][0].split('|')[2])\n", - " data_['lut_rel'] = float(lines[np.array(['CLB LUTs*' in line for line in lines])][0].split('|')[5])\n", - " data_['ff_rel'] = float(lines[np.array(['CLB Registers' in line for line in lines])][0].split('|')[5])\n", - " data_['bram_rel'] = float(lines[np.array(['Block RAM Tile' in line for line in lines])][0].split('|')[5])\n", - " data_['dsp_rel'] = float(lines[np.array(['DSPs' in line for line in lines])][0].split('|')[5])\n", - "\n", - " with report_csynth.open() as report:\n", - " lines = np.array(report.readlines())\n", - " lat_line = lines[np.argwhere(np.array(['Latency (cycles)' in line for line in lines])).flatten()[0] + 3]\n", - " data_['latency_clks'] = int(lat_line.split('|')[2])\n", - " data_['latency_mus'] = float(lat_line.split('|')[2]) * 5.0 / 1000.0\n", - " data_['latency_ii'] = int(lat_line.split('|')[6])\n", - "\n", - " return data_" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import pprint\n", - "\n", - "data_pruned_ref = getReports('pruned_cnn')\n", - "data_quantized_pruned = getReports('quantized_pruned_cnn')\n", - "\n", - "print(\"\\n Resource usage and latency: Pruned\")\n", - "pprint.pprint(data_pruned_ref)\n", - "print(\"\\n Resource usage and latency: Pruned + quantized\")\n", - "pprint.pprint(data_quantized_pruned)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We see that the latency is of around 5 microseconds for both the quantized and the unquantized model, but that the resources are signifcantly reduced using QKeras.\n", - "\n", - "Congratulations! You have now reached the end of this notebook. If you have some spare time, you can have a look at the bonus exercise below, where you will learn how to perform a bayesian optimization over the QKeras quantizers in order to obtain an optimally heterogeneously quantized model.\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "deletable": false, - "editable": false - }, - "source": [ - "## Bonus exercise: Automatic quantization with AutoQKeras\n", - "\n", - "In this bonus exercise, you will learn how to find the optimal heterogeneously quantized model using AutoQKeras.\n", - "For more details, you can look at the [AutoQKeras notebook](https://github.com/google/qkeras/blob/master/notebook/AutoQKeras.ipynb). \n", - "\n", - "Let's first check the estimated energy consumption of the QKeras 6-bit model using QTools. By setting ```for_reference=True``` you can print out the unquantized model energy consumption and compare the two. Note that this only works for QKeras layers. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "filters_per_conv_layer = [16, 16, 24]\n", - "neurons_per_dense_layer = [42, 64]\n", - "\n", - "x = x_in = Input(input_shape)\n", - "\n", - "for i, f in enumerate(filters_per_conv_layer):\n", - " print(('Adding convolutional block {} with N={} filters').format(i, f))\n", - " x = Conv2D(\n", - " int(f),\n", - " kernel_size=(3, 3),\n", - " strides=(1, 1),\n", - " kernel_initializer='lecun_uniform',\n", - " kernel_regularizer=l1(0.0001),\n", - " use_bias=False,\n", - " name='conv_{}'.format(i),\n", - " )(x)\n", - " x = BatchNormalization(name='bn_conv_{}'.format(i))(x)\n", - " x = Activation('relu', name='conv_act_%i' % i)(x)\n", - " x = MaxPooling2D(pool_size=(2, 2), name='pool_{}'.format(i))(x)\n", - "x = Flatten()(x)\n", - "\n", - "for i, n in enumerate(neurons_per_dense_layer):\n", - " print(('Adding dense block {} with N={} neurons').format(i, n))\n", - " x = Dense(n, kernel_initializer='lecun_uniform', kernel_regularizer=l1(0.0001), name='dense_%i' % i, use_bias=False)(x)\n", - " x = BatchNormalization(name='bn_dense_{}'.format(i))(x)\n", - " x = Activation('relu', name='dense_act_%i' % i)(x)\n", - "x = Dense(int(n_classes), name='output_dense')(x)\n", - "x_out = Activation('softmax', name='output_softmax')(x)\n", - "\n", - "baseline_model = Model(inputs=[x_in], outputs=[x_out], name='keras_baseline')\n", - "\n", - "LOSS = tf.keras.losses.CategoricalCrossentropy()\n", - "OPTIMIZER = tf.keras.optimizers.Adam(learning_rate=3e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=True)\n", - "\n", - "baseline_model.compile(loss=LOSS, optimizer=OPTIMIZER, metrics=[\"accuracy\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from qkeras import print_qstats\n", - "\n", - "# for automatic quantization\n", - "import pprint\n", - "from qkeras.autoqkeras import *\n", - "from qkeras import *\n", - "from qkeras.utils import model_quantize\n", - "\n", - "from qkeras.qtools import run_qtools\n", - "from qkeras.qtools import settings as qtools_settings\n", - "from tensorflow_model_optimization.python.core.sparsity.keras import pruning_wrapper\n", - "from qkeras import quantized_bits\n", - "from qkeras import QDense, QActivation\n", - "\n", - "q = run_qtools.QTools(\n", - " baseline_model,\n", - " process=\"horowitz\",\n", - " source_quantizers=[quantized_bits(16, 5, 1)],\n", - " is_inference=True,\n", - " weights_path=None,\n", - " keras_quantizer=\"fp16\",\n", - " keras_accumulator=\"fp16\",\n", - " for_reference=False,\n", - ")\n", - "q.qtools_stats_print()\n", - "\n", - "energy_dict = q.pe(\n", - " weights_on_memory=\"fixed\", activations_on_memory=\"fixed\", min_sram_size=8 * 16 * 1024 * 1024, rd_wr_on_io=False\n", - ")\n", - "\n", - "# get stats of energy distribution in each layer\n", - "energy_profile = q.extract_energy_profile(qtools_settings.cfg.include_energy, energy_dict)\n", - "# extract sum of energy of each layer according to the rule specified in\n", - "# qtools_settings.cfg.include_energy\n", - "total_energy = q.extract_energy_sum(qtools_settings.cfg.include_energy, energy_dict)\n", - "\n", - "pprint.pprint(energy_profile)\n", - "print()\n", - "\n", - "print(\"Total energy: {:.6f} uJ\".format(total_energy / 1000000.0))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, lets use AutoQKeras to find an optimally heterogeneously quantized model for us. For more details, check the AutoQKeras tutorial linked above. As baseline model, we'll use the pruned floating point Keras model from above." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# These are the quantizers we'll test in the bayesian optimization\n", - "quantization_config = {\n", - " \"kernel\": {\n", - " \"quantized_bits(2,0,1,alpha=1.0)\": 2,\n", - " \"quantized_bits(4,0,1,alpha=1.0)\": 4,\n", - " \"quantized_bits(6,0,1,alpha=1.0)\": 6,\n", - " \"quantized_bits(8,0,1,alpha=1.0)\": 8,\n", - " },\n", - " \"bias\": {\n", - " \"quantized_bits(2,0,1,alpha=1.0)\": 2,\n", - " \"quantized_bits(4,0,1,alpha=1.0)\": 4,\n", - " \"quantized_bits(6,0,1,alpha=1.0)\": 6,\n", - " \"quantized_bits(8,0,1,alpha=1.0)\": 8,\n", - " },\n", - " \"activation\": {\n", - " \"quantized_relu(3,1)\": 3,\n", - " \"quantized_relu(4,2)\": 4,\n", - " \"quantized_relu(8,2)\": 8,\n", - " \"quantized_relu(8,4)\": 8,\n", - " \"quantized_relu(16,6)\": 16,\n", - " },\n", - " \"linear\": {\n", - " \"quantized_bits(2,0,1,alpha=1.0)\": 2,\n", - " \"quantized_bits(4,0,1,alpha=1.0)\": 4,\n", - " \"quantized_bits(6,0,1,alpha=1.0)\": 6,\n", - " \"quantized_bits(8,0,1,alpha=1.0)\": 8,\n", - " },\n", - "}\n", - "\n", - "# These are the layer types we will quantize\n", - "limit = {\n", - " \"Dense\": [8, 8, 16],\n", - " \"Conv2D\": [8, 8, 16],\n", - " \"Activation\": [16],\n", - "}\n", - "\n", - "# Use this if you want to minimize the model bit size\n", - "goal_bits = {\n", - " \"type\": \"bits\",\n", - " \"params\": {\n", - " \"delta_p\": 8.0, # We tolerate up to a +8% accuracy change\n", - " \"delta_n\": 8.0, # We tolerate down to a -8% accuracy change\n", - " \"rate\": 2.0, # We want a x2 times smaller model\n", - " \"stress\": 1.0, # Force the reference model size to be smaller by setting stress<1\n", - " \"input_bits\": 8,\n", - " \"output_bits\": 8,\n", - " \"ref_bits\": 8,\n", - " \"config\": {\"default\": [\"parameters\", \"activations\"]},\n", - " },\n", - "}\n", - "\n", - "# Use this if you want to minimize the model energy consumption\n", - "goal_energy = {\n", - " \"type\": \"energy\",\n", - " \"params\": {\n", - " \"delta_p\": 8.0,\n", - " \"delta_n\": 8.0,\n", - " \"rate\": 2.0,\n", - " \"stress\": 1.0,\n", - " \"process\": \"horowitz\",\n", - " \"parameters_on_memory\": [\"sram\", \"sram\"],\n", - " \"activations_on_memory\": [\"sram\", \"sram\"],\n", - " \"rd_wr_on_io\": [False, False],\n", - " \"min_sram_size\": [0, 0],\n", - " \"source_quantizers\": [\"fp32\"],\n", - " \"reference_internal\": \"int8\",\n", - " \"reference_accumulator\": \"int32\",\n", - " },\n", - "}\n", - "\n", - "run_config = {\n", - " \"goal\": goal_energy,\n", - " \"quantization_config\": quantization_config,\n", - " \"learning_rate_optimizer\": False,\n", - " \"transfer_weights\": False, # Randomely initialize weights\n", - " \"mode\": \"bayesian\", # This can be bayesian,random,hyperband\n", - " \"seed\": 42,\n", - " \"limit\": limit,\n", - " \"tune_filters\": \"layer\",\n", - " \"tune_filters_exceptions\": \"^output\",\n", - " \"distribution_strategy\": None,\n", - " \"max_trials\": 5, # Let's just do 5 trials for this demonstrator, ideally you should do as many as possible\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from qkeras.autoqkeras import AutoQKeras\n", - "\n", - "autoqk = AutoQKeras(baseline_model, output_dir=\"autoq_cnn\", metrics=[\"acc\"], custom_objects={}, **run_config)\n", - "autoqk.fit(train_data, validation_data=val_data, epochs=15)\n", - "\n", - "aqmodel = autoqk.get_best_model()\n", - "print_qmodel_summary(aqmodel)\n", - "\n", - "# Train for the full epochs\n", - "callbacks = [\n", - " tf.keras.callbacks.EarlyStopping(patience=10, verbose=1),\n", - " tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, verbose=1),\n", - "]\n", - "\n", - "start = time.time()\n", - "history = aqmodel.fit(train_data, epochs=n_epochs, validation_data=val_data, callbacks=callbacks, verbose=1)\n", - "end = time.time()\n", - "print('\\n It took {} minutes to train!\\n'.format((end - start) / 60.0))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# This model has some remnants from the optimization procedure attached to it, so let's define a new one\n", - "aqmodel.save_weights(\"autoqkeras_cnn_weights.h5\")\n", - "\n", - "layers = [l for l in aqmodel.layers]\n", - "x = layers[0].output\n", - "for i in range(1, len(layers)):\n", - " x = layers[i](x)\n", - "\n", - "new_model = Model(inputs=[layers[0].input], outputs=[x])\n", - "LOSS = tf.keras.losses.CategoricalCrossentropy()\n", - "OPTIMIZER = tf.keras.optimizers.Adam(learning_rate=3e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=True)\n", - "\n", - "new_model.compile(loss=LOSS, optimizer=OPTIMIZER, metrics=[\"accuracy\"])\n", - "new_model.summary()\n", - "new_model.load_weights(\"autoqkeras_cnn_weights.h5\")\n", - "print_qmodel_summary(new_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's check what the best heterogeneously quantized model looks like (keep in mind we only did a few trials, the optimization obviosuly didn't have time to converge at the minimum but yo get the idea!)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hls_config_aq = hls4ml.utils.config_from_keras_model(new_model, granularity='name')\n", - "hls_config_aq['Model']['ReuseFactor'] = 1\n", - "hls_config_aq['Model']['Precision'] = 'ap_fixed<16,6>'\n", - "hls_config_aq['LayerName']['output_softmax']['Strategy'] = 'Stable'\n", - "plotting.print_dict(hls_config_aq)\n", - "\n", - "cfg_aq = hls4ml.converters.create_config(backend='Vivado')\n", - "cfg_aq['IOType'] = 'io_stream' # Must set this if using CNNs!\n", - "cfg_aq['HLSConfig'] = hls_config_aq\n", - "cfg_aq['KerasModel'] = new_model\n", - "cfg_aq['OutputDir'] = 'autoqkeras_cnn/'\n", - "cfg_aq['XilinxPart'] = 'xcu250-figd2104-2L-e'\n", - "\n", - "hls_model_aq = hls4ml.converters.keras_to_hls(cfg_aq)\n", - "hls_model_aq.compile()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "y_predict_aq = aqmodel.predict(X_test_reduced)\n", - "y_predict_hls4ml_aq = hls_model_aq.predict(np.ascontiguousarray(X_test_reduced))\n", - "\n", - "\n", - "accuracy_keras = float(accuracy_score(np.argmax(Y_test_reduced, axis=1), np.argmax(y_predict_aq, axis=1)))\n", - "accuracy_hls4ml = float(accuracy_score(np.argmax(Y_test_reduced, axis=1), np.argmax(y_predict_hls4ml_aq, axis=1)))\n", - "\n", - "print(\"Accuracy AutoQ Keras: {}\".format(accuracy_keras))\n", - "print(\"Accuracy AutoQ hls4ml: {}\".format(accuracy_hls4ml))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The accuracy is slightly lower for this heterogeneously quantized model. Due to some randomness in the optimization procedure, you're going to have to synthesize this one yourself!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "synth = True\n", - "if synth:\n", - " hls_model_aq.build(csim=False, synth=True, vsynth=True)\n", - " data_autoq = getReports('autoq_cnn')\n", - "\n", - " print(\"\\n Resource usage and latency: AutoQ\")\n", - " pprint.pprint(data_autoq)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/part8_symbolic_regression.ipynb b/part8_symbolic_regression.ipynb deleted file mode 100644 index c46578e3..00000000 --- a/part8_symbolic_regression.ipynb +++ /dev/null @@ -1,500 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "79933ff7", - "metadata": {}, - "source": [ - "# Part 8: Symbolic Regression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ede2226f", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import sympy\n", - "import matplotlib.pyplot as plt\n", - "import hls4ml\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.preprocessing import StandardScaler, LabelEncoder\n", - "from sklearn.metrics import roc_curve, auc, accuracy_score\n", - "from tensorflow.keras.utils import to_categorical\n", - "from sklearn.datasets import fetch_openml" - ] - }, - { - "cell_type": "markdown", - "id": "d9e2b159", - "metadata": {}, - "source": [ - "## Load the LHC jet tagging dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ee6d96bd", - "metadata": {}, - "outputs": [], - "source": [ - "data = fetch_openml('hls4ml_lhc_jets_hlf')\n", - "X, Y = data['data'].to_numpy(), data['target'].to_numpy()\n", - "print(data['feature_names'])\n", - "print(X.shape, Y.shape)\n", - "print(Y[:10])\n", - "\n", - "LE = LabelEncoder()\n", - "Y = LE.fit_transform(Y)\n", - "Y = to_categorical(Y, 5)\n", - "\n", - "Y = 2 * Y - 1\n", - "print(Y[:10])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0502aea8", - "metadata": {}, - "outputs": [], - "source": [ - "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.5, random_state=123)\n", - "\n", - "scaler = StandardScaler().fit(X_train)\n", - "X_train = scaler.transform(X_train)\n", - "X_test = scaler.transform(X_test)\n", - "\n", - "# PySR (or any genetic programming based SR) not happy with too many training data\n", - "X_train = X_train[:8000]\n", - "Y_train = Y_train[:8000]\n", - "\n", - "print('X_train.shape: ' + str(X_train.shape))\n", - "print('Y_train.shape: ' + str(Y_train.shape))\n", - "print('X_test.shape: ' + str(X_test.shape))\n", - "print('Y_test.shape: ' + str(Y_test.shape))" - ] - }, - { - "cell_type": "markdown", - "id": "7ec86106", - "metadata": {}, - "source": [ - "## Perform SR with PySR (if installed)" - ] - }, - { - "cell_type": "markdown", - "id": "57e7896d", - "metadata": {}, - "source": [ - "If you want to run `PySR` (a genetic programming-based symbolic regression software), please see https://github.com/MilesCranmer/PySR for installation and intructions.\n", - "\n", - "Below is an example configuration script to run training in `PySR`, where one can specify the allowed primitive functions `unary_operators` `binary_operators` (e.g. `+`, `*`, `sin`) and constraints `complexity_of_operators` `constraints` `nested_constraints` in the equation seacrhing. The training results will be stored in a `.pkl` file that contains the final equations selected by the training strategy `model_selection`.\n", - "\n", - "We also provide an already trained PySR model `sr/example.pkl` in the following sections for demonstrating the HLS implementation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "96a651dd", - "metadata": {}, - "outputs": [], - "source": [ - "from pysr import PySRRegressor\n", - "\n", - "!export JULIA_NUM_THREADS=32\n", - "\n", - "model_pysr = PySRRegressor(\n", - " model_selection='accuracy',\n", - " niterations=40,\n", - " timeout_in_seconds=60 * 60 * 1,\n", - " maxsize=40,\n", - " select_k_features=6,\n", - " binary_operators=['+', '-', '*'],\n", - " unary_operators=['sin', 'sc(x)=sin(x)*cos(x)'],\n", - " complexity_of_operators={'+': 1, '-': 1, '*': 1, 'sin': 1, 'sc': 1},\n", - " constraints={'sin': 20, 'sc': 20},\n", - " nested_constraints={'sin': {'sin': 0, 'sc': 0}, 'sc': {'sin': 0, 'sc': 0}},\n", - " extra_sympy_mappings={'sc': lambda x: sympy.sin(x) * sympy.cos(x)},\n", - " loss='L2MarginLoss()', # (1 - y*y')^2\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5f4d9501", - "metadata": {}, - "outputs": [], - "source": [ - "model_pysr.fit(X_train, Y_train)" - ] - }, - { - "cell_type": "markdown", - "id": "846e710b", - "metadata": {}, - "source": [ - "## Prepare symbolic expressions in strings first" - ] - }, - { - "cell_type": "markdown", - "id": "c7aaf105", - "metadata": {}, - "source": [ - "We provide a trained model for the HLS demonstration.\n", - "\n", - "**If you have `PySR` installed**, you can directly load the trained expressions from the output file `sr/example.pkl`.\n", - "`PySR` allows custom functions to be defined, such as sc(x):=sin(x)*cos(x) in this example, they need to be re-defined through `extra_sympy_mappings` and a new `sympy` class when retrieving the equations for evaluation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d3d5d2cd", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "from pysr import PySRRegressor\n", - "\n", - "model_pysr = PySRRegressor.from_file('sr/example.pkl')\n", - "with sympy.evaluate(True):\n", - " for i in range(5):\n", - " print('Tagger {} = '.format(i) + str(model_pysr.sympy()[i]) + '\\n------------------------------------------')\n", - "\n", - "# Re-write custom operator defined from PySR config: sc(x) = sin(x)*cos(x)\n", - "model_pysr.set_params(extra_sympy_mappings={\"sc\": lambda x: sympy.sin(x) * sympy.cos(x)})\n", - "model_pysr.refresh()\n", - "\n", - "\n", - "class sc(sympy.Function):\n", - " pass" - ] - }, - { - "cell_type": "markdown", - "id": "699d2e05", - "metadata": {}, - "source": [ - "There are two options for evaluating math functions in `hls4ml`, one is using the standard HLS math library (`func`), another one is using approximation with user-defined lookup tables (`func_lut`) for resources saving. We will define the lookup tables (table range and size) for `func_lut` later.\n", - "\n", - "We have the equations in the `sympy` format, now convert them into strings: `expr` for using the standard functions and `expr_lut` for using the approximation with lookup tables. We will re-parse `expr` and `expr_lut` from strings in `sympy` format for the `hls4ml` converter." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7219a874", - "metadata": {}, - "outputs": [], - "source": [ - "expr = []\n", - "expr_lut = []\n", - "for i in range(5):\n", - " expr.append(str(model_pysr.sympy()[i]))\n", - " expr_lut.append(expr[i].replace(\"sin\", \"sin_lut\").replace(\"cos\", \"cos_lut\"))" - ] - }, - { - "cell_type": "markdown", - "id": "0abcba26", - "metadata": {}, - "source": [ - "**If you don't have PySR installed**, you can also write your expressions directly in strings and parse in `sympy` format, which can then be fed to `hls4ml` converter. Here again, `expr` for using standard math library, `expr_lut` for using approximation with lookup tables." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3356d1e6", - "metadata": {}, - "outputs": [], - "source": [ - "# Expressions from 'sr/example.pkl'\n", - "\n", - "# Expressions that will use Vivado math library\n", - "expr = [\n", - " '-0.1630426*(sin(-0.75052315)*cos(-0.75052315) - 0.84283006)*sin(2*x14 - 1.03665108)*cos(2*x14 - 1.03665108) - sin(x14 - (0.9237657 - 0.11933863*x3)*(-x15 + 2*x2 - 0.3817056) + 1.761264957)',\n", - " '(-(0.5822144*sin(0.83811*x14)*cos(0.83811*x14) - 0.5324657)*(sin(0.3923645*x2)*cos(0.3923645*x2) - 0.63548696) + sin(x14 - 0.3923645*x15 + x3 + 0.51168373)*cos(x14 - 0.3923645*x15 + x3 + 0.51168373))*(0.561041303633489*sin(x15) - 0.47277835) - 0.84055585',\n", - " '0.49239117*(sin(x3)*cos(x3) + sin(x15 + 0.76784414*x3)*cos(x15 + 0.76784414*x3))*(sin(-0.13417026)*cos(-0.13417026) + sin(0.5180547)*cos(0.5180547) + sin(x2)*cos(x2)) - sin(x14 + 0.25715914*x15*x3 - x2 - x3 + 0.66443527)',\n", - " '0.41071504*(0.9298677 - sin(0.59376544*x15))*(sin(x14)*cos(x14) + 5.2546763*sin(0.71913457 - x3)*cos(0.71913457 - x3))*(-sin(2*x3)*cos(2*x3) + sin(5.2546763*x14 + x3 + 0.77032656)*cos(5.2546763*x14 + x3 + 0.77032656) + 0.32492808) - 0.863786752431664',\n", - " '(1.0745832 - sin(-x14 - 0.4094719)*cos(-x14 - 0.4094719))*(-0.15737492*x15 - sin(x14 - 4.2594776)*cos(x14 - 4.2594776) + sin(3*x14 - x3*(x14 - 4.1772995) - x3 + 3.087878)*cos(3*x14 - x3*(x14 - 4.1772995) - x3 + 3.087878) - 0.690204005690814)',\n", - "]\n", - "# Expressions that will use look-up table approximated math functions\n", - "expr_lut = []\n", - "for i in range(len(expr)):\n", - " expr_lut.append(expr[i].replace(\"sin\", \"sin_lut\").replace(\"cos\", \"cos_lut\"))" - ] - }, - { - "cell_type": "markdown", - "id": "788ee608", - "metadata": {}, - "source": [ - "## Then parse the strings to sympy expressions" - ] - }, - { - "cell_type": "markdown", - "id": "03fc8284", - "metadata": {}, - "source": [ - "Define the lookup tables for approximating math functions. The table range and size can be customized for each function to be approximated, they depend on how much precision can be compromised to save more resources." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "920e2326", - "metadata": {}, - "outputs": [], - "source": [ - "from hls4ml.utils.symbolic_utils import init_pysr_lut_functions\n", - "\n", - "# For functions approximated with look-up table, define the table range and size\n", - "function_definitions = [\n", - " 'sin_lut(x) = math_lut(sin, x, N=256, range_start=-8, range_end=8)',\n", - " 'cos_lut(x) = math_lut(cos, x, N=256, range_start=-8, range_end=8)',\n", - "]\n", - "init_pysr_lut_functions(init_defaults=True, function_definitions=function_definitions)\n", - "\n", - "lut_functions = {\n", - " 'sin_lut': {'math_func': 'sin', 'range_start': -8, 'range_end': 8, 'table_size': 256},\n", - " 'cos_lut': {'math_func': 'cos', 'range_start': -8, 'range_end': 8, 'table_size': 256},\n", - "}" - ] - }, - { - "cell_type": "markdown", - "id": "8be93891", - "metadata": {}, - "source": [ - "Parse `expr` and `expr_lut` to sympy expressions." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "96f61066", - "metadata": {}, - "outputs": [], - "source": [ - "# Use sympy to parse strings into sympy expressions\n", - "for i in range(len(expr)):\n", - " print('expr =\\n' + expr[i])\n", - " print(\"----------------------------------------\")\n", - " print('expr_LUT =\\n' + expr_lut[i])\n", - " print(\"========================================\")\n", - " expr[i] = sympy.parsing.sympy_parser.parse_expr(expr[i])\n", - " expr_lut[i] = sympy.parsing.sympy_parser.parse_expr(expr_lut[i])" - ] - }, - { - "cell_type": "markdown", - "id": "f7548c93", - "metadata": {}, - "source": [ - "Use `hls4ml.converters.convert_from_symbolic_expression` to convert sympy expressions and compile." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "46ff4b5e", - "metadata": {}, - "outputs": [], - "source": [ - "# Use hls4ml to convert sympy expressions into HLS model\n", - "hls_model = hls4ml.converters.convert_from_symbolic_expression(\n", - " expr, n_symbols=16, output_dir='my-hls-test', precision='ap_fixed<16,6>', part='xcvu9p-flga2577-2-e'\n", - ")\n", - "hls_model.write()\n", - "hls_model.compile()\n", - "\n", - "hls_model_lut = hls4ml.converters.convert_from_symbolic_expression(\n", - " expr_lut,\n", - " n_symbols=16,\n", - " output_dir='my-hls-test-lut',\n", - " precision='ap_fixed<16,6>',\n", - " part='xcvu9p-flga2577-2-e',\n", - " lut_functions=lut_functions,\n", - ")\n", - "hls_model_lut.write()\n", - "hls_model_lut.compile()" - ] - }, - { - "cell_type": "markdown", - "id": "08682628", - "metadata": {}, - "source": [ - "## Compare outputs: PySR vs HLS vs HLS(LUT)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "39269441", - "metadata": {}, - "outputs": [], - "source": [ - "test_vector = np.random.rand(1, 16) * 4 - 2\n", - "# print(model_pysr.predict(test_vector))\n", - "print(hls_model.predict(test_vector))\n", - "print(hls_model_lut.predict(test_vector))" - ] - }, - { - "cell_type": "markdown", - "id": "08795fca", - "metadata": {}, - "source": [ - "## Compare performance on the dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "05894f0b", - "metadata": {}, - "outputs": [], - "source": [ - "# Y_pysr = model_pysr.predict(X_test)\n", - "Y_hls = hls_model.predict(X_test)\n", - "Y_hls_lut = hls_model_lut.predict(X_test)\n", - "# auc_pysr=[]\n", - "auc_hls = []\n", - "auc_hls_lut = []\n", - "for x, label in enumerate(LE.classes_):\n", - " # fpr_pysr, tpr_pysr, _ = roc_curve(Y_test[:, x], Y_pysr[:, x])\n", - " fpr_hls, tpr_hls, _ = roc_curve(Y_test[:, x], Y_hls[:, x])\n", - " fpr_hls_lut, tpr_hls_lut, _ = roc_curve(Y_test[:, x], Y_hls_lut[:, x])\n", - " # auc_pysr.append(auc(fpr_pysr, tpr_pysr))\n", - " auc_hls.append(auc(fpr_hls, tpr_hls))\n", - " auc_hls_lut.append(auc(fpr_hls_lut, tpr_hls_lut))\n", - "\n", - "# print('PySR acc = {0:.3f}'.format(accuracy_score(np.argmax(Y_test, axis=1), np.argmax(Y_pysr, axis=1))))\n", - "# print('PySR auc = {0:.3f},{1:.3f},{2:.3f},{3:.3f},{4:.3f}'.format(auc_pysr[0],auc_pysr[1],auc_pysr[2],auc_pysr[3],auc_pysr[4]))\n", - "print('HLS acc = {0:.3f}'.format(accuracy_score(np.argmax(Y_test, axis=1), np.argmax(Y_hls, axis=1))))\n", - "print(\n", - " 'HLS auc = {0:.3f},{1:.3f},{2:.3f},{3:.3f},{4:.3f}'.format(\n", - " auc_hls[0], auc_hls[1], auc_hls[2], auc_hls[3], auc_hls[4]\n", - " )\n", - ")\n", - "print('HLS_LUT acc = {0:.3f}'.format(accuracy_score(np.argmax(Y_test, axis=1), np.argmax(Y_hls_lut, axis=1))))\n", - "print(\n", - " 'HLS_LUT auc = {0:.3f},{1:.3f},{2:.3f},{3:.3f},{4:.3f}'.format(\n", - " auc_hls_lut[0], auc_hls_lut[1], auc_hls_lut[2], auc_hls_lut[3], auc_hls_lut[4]\n", - " )\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "002643a3", - "metadata": {}, - "outputs": [], - "source": [ - "def plot_roc(y_test, y_pred, labels, model):\n", - " color = ['blue', 'orange', 'green', 'red', 'purple']\n", - " for x, label in enumerate(labels):\n", - " fpr, tpr, _ = roc_curve(y_test[:, x], y_pred[:, x])\n", - " if model == 'pysr':\n", - " plt.plot(\n", - " tpr,\n", - " fpr,\n", - " label='{0}, PySR, AUC = {1:.1f}'.format(label, auc(fpr, tpr) * 100.0),\n", - " linestyle='solid',\n", - " color=color[x],\n", - " lw=1.5,\n", - " )\n", - " if model == 'hls':\n", - " plt.plot(\n", - " tpr,\n", - " fpr,\n", - " label='{0}, HLS, AUC = {1:.1f}'.format(label, auc(fpr, tpr) * 100.0),\n", - " linestyle='dotted',\n", - " color=color[x],\n", - " lw=1.5,\n", - " )\n", - " if model == 'hls_lut':\n", - " plt.plot(\n", - " tpr,\n", - " fpr,\n", - " label='{0}, HLS LUT, AUC = {1:.1f}'.format(label, auc(fpr, tpr) * 100.0),\n", - " linestyle='None',\n", - " color=color[x],\n", - " lw=1,\n", - " marker='o',\n", - " ms=1,\n", - " )\n", - " plt.semilogy()\n", - " plt.xlabel('True positive rate', size=15, loc='right')\n", - " plt.ylabel('False positive rate', size=15, loc='top')\n", - " plt.tick_params(axis='both', which='major', direction='in', length=6, width=1.2, labelsize=12, right=True, top=True)\n", - " plt.tick_params(axis='both', which='minor', direction='in', length=2, width=1, labelsize=12, right=True, top=True)\n", - " plt.xlim(0, 1)\n", - " plt.ylim(0.001, 1)\n", - " plt.grid(True)\n", - " plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), fontsize=12)\n", - "\n", - "\n", - "plt.figure(figsize=(15, 15))\n", - "axes = plt.subplot(2, 2, 1)\n", - "# plot_roc(Y_test, Y_pysr, LE.classes_, 'pysr')\n", - "plot_roc(Y_test, Y_hls, LE.classes_, 'hls')\n", - "plot_roc(Y_test, Y_hls_lut, LE.classes_, 'hls_lut')" - ] - }, - { - "cell_type": "markdown", - "id": "7beb92ea", - "metadata": {}, - "source": [ - "## Run synthesis from command line" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e4047f52", - "metadata": {}, - "outputs": [], - "source": [ - "!source ${XILINX_VITIS}/settings64.sh\n", - "!vitis_hls -f build_prj.tcl \"reset=1 synth=1 csim=0 cosim=0 validation=0 export=0 vsynth=0\"\n", - "!cat my-hls-test/myproject_prj/solution1/syn/report/myproject_csynth.rpt" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/sr/example.pkl b/sr/example.pkl deleted file mode 100644 index d3fecff7..00000000 Binary files a/sr/example.pkl and /dev/null differ