{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "EDdA-Classification_CNN_Conv1D-EGC.ipynb",
"provenance": [],
"collapsed_sections": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "0yFsoHXX8Iyy"
},
"source": [
"# Deep learning for EDdA classification"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "tFlUCDL2778i"
},
"source": [
"## Setup colab environment"
]
},
{
"cell_type": "code",
"metadata": {
"id": "Sp8d_Uus7SHJ",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "976ed0dd-7aeb-4f64-e34b-117733abf38c"
},
"source": [
"from google.colab import drive\n",
"drive.mount('/content/drive')"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Mounted at /content/drive\n"
]
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "jQBu-p6hBU-j"
},
"source": [
"### Install packages"
]
},
{
"cell_type": "code",
"metadata": {
"id": "bTIXsF6kBUdh"
},
"source": [
"#!pip install zeugma\n",
"#!pip install plot_model"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "56-04SNF8BMx"
},
"source": [
"### Import librairies"
]
},
{
"cell_type": "code",
"metadata": {
"id": "HwWkSznz7SEv"
},
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import pickle\n",
"import os\n",
"\n",
"from tqdm import tqdm\n",
"import requests, zipfile, io\n",
"import codecs\n",
"\n",
"from sklearn import preprocessing # LabelEncoder\n",
"from sklearn.metrics import classification_report\n",
"from sklearn.metrics import confusion_matrix\n",
"\n",
"from keras.preprocessing import sequence\n",
"from keras.preprocessing.text import Tokenizer\n",
"\n",
"from keras.layers import BatchNormalization, Input, Reshape, Conv1D, MaxPool1D, Conv2D, MaxPool2D, Concatenate\n",
"from keras.layers import Embedding, Dropout, Flatten, Dense\n",
"from keras.models import Model, load_model\n",
"from keras.callbacks import ModelCheckpoint\n"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "xrekV6W978l4"
},
"source": [
"### Utils functions"
]
},
{
"cell_type": "code",
"metadata": {
"id": "4LJ5blQR7PUe"
},
"source": [
"\n",
"def resample_classes(df, classColumnName, numberOfInstances):\n",
" #random numberOfInstances elements\n",
" replace = False # with replacement\n",
" fn = lambda obj: obj.loc[np.random.choice(obj.index, numberOfInstances if len(obj) > numberOfInstances else len(obj), replace),:]\n",
" return df.groupby(classColumnName, as_index=False).apply(fn)\n",
" \n"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "MtLr35eM753e"
},
"source": [
"## Load Data"
]
},
{
"cell_type": "code",
"metadata": {
"id": "FnbNT4NF7zal",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "c2a72d94-c7ae-4e6a-b962-ec4677053555"
},
"source": [
"!wget https://projet.liris.cnrs.fr/geode/EDdA-Classification/datasets/training_set.tsv\n",
"!wget https://projet.liris.cnrs.fr/geode/EDdA-Classification/datasets/test_set.tsv"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"--2022-02-17 19:08:55-- https://projet.liris.cnrs.fr/geode/EDdA-Classification/datasets/training_set.tsv\n",
"Resolving projet.liris.cnrs.fr (projet.liris.cnrs.fr)... 134.214.142.28\n",
"Connecting to projet.liris.cnrs.fr (projet.liris.cnrs.fr)|134.214.142.28|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 175634219 (167M) [text/tab-separated-values]\n",
"Saving to: ‘training_set.tsv’\n",
"\n",
"training_set.tsv 100%[===================>] 167.50M 28.2MB/s in 6.5s \n",
"\n",
"2022-02-17 19:09:02 (25.7 MB/s) - ‘training_set.tsv’ saved [175634219/175634219]\n",
"\n",
"--2022-02-17 19:09:02-- https://projet.liris.cnrs.fr/geode/EDdA-Classification/datasets/test_set.tsv\n",
"Resolving projet.liris.cnrs.fr (projet.liris.cnrs.fr)... 134.214.142.28\n",
"Connecting to projet.liris.cnrs.fr (projet.liris.cnrs.fr)|134.214.142.28|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 42730598 (41M) [text/tab-separated-values]\n",
"Saving to: ‘test_set.tsv’\n",
"\n",
"test_set.tsv 100%[===================>] 40.75M 19.7MB/s in 2.1s \n",
"\n",
"2022-02-17 19:09:05 (19.7 MB/s) - ‘test_set.tsv’ saved [42730598/42730598]\n",
"\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"### Loading dataset"
],
"metadata": {
"id": "UHushJ1XfUj9"
}
},
{
"cell_type": "code",
"source": [
"train_path = 'training_set.tsv'\n",
"test_path = 'test_set.tsv'"
],
"metadata": {
"id": "Q4te2c0bfvaJ"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "nRLaQUO97zcq"
},
"source": [
"df_train = pd.read_csv(train_path, sep=\"\\t\")\n"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"df_train.sample(5)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 548
},
"id": "2MvHEc7zVK1N",
"outputId": "a934bbce-4ebe-4d5a-eb4e-18db1ce10532"
},
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/html": [
"\n",
" <div id=\"df-24c25d4d-881e-4c52-8629-da71863f5656\">\n",
" <div class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>volume</th>\n",
" <th>numero</th>\n",
" <th>head</th>\n",
" <th>normClass</th>\n",
" <th>classEDdA</th>\n",
" <th>author</th>\n",
" <th>id_enccre</th>\n",
" <th>domaine_enccre</th>\n",
" <th>ensemble_domaine_enccre</th>\n",
" <th>content</th>\n",
" <th>contentWithoutClass</th>\n",
" <th>firstParagraph</th>\n",
" <th>nb_words</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>17253</th>\n",
" <td>1</td>\n",
" <td>4938</td>\n",
" <td>Auge</td>\n",
" <td>Verrerie</td>\n",
" <td>dans les Verreries</td>\n",
" <td>unsigned</td>\n",
" <td>v1-3613-9</td>\n",
" <td>verrerie</td>\n",
" <td>Métiers</td>\n",
" <td>Auge, dans les Verreries, ce sont de gros hêtr...</td>\n",
" <td>auge gros hêtre \\n creusés tient plein eau ser...</td>\n",
" <td>auge gros hêtre \\n creusés tient plein eau ser...</td>\n",
" <td>70</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16170</th>\n",
" <td>9</td>\n",
" <td>2017</td>\n",
" <td>Lettres de reprise</td>\n",
" <td>unclassified</td>\n",
" <td>unclassified</td>\n",
" <td>Boucher d'Argis</td>\n",
" <td>v9-1324-120</td>\n",
" <td>jurisprudence</td>\n",
" <td>Droit - Jurisprudence</td>\n",
" <td>Lettres de reprise, sont une commission que\\nl...</td>\n",
" <td>lettre reprise commission \\n prend chancelleri...</td>\n",
" <td>lettre reprise commission \\n prend chancelleri...</td>\n",
" <td>36</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12041</th>\n",
" <td>16</td>\n",
" <td>1213</td>\n",
" <td>THOLUS</td>\n",
" <td>Architecture romaine</td>\n",
" <td>Archit. rom.</td>\n",
" <td>Jaucourt</td>\n",
" <td>v16-709-0</td>\n",
" <td>architecture</td>\n",
" <td>Architecture</td>\n",
" <td>THOLUS, s. m. (Archit. rom.) Vitruve nomme\\nth...</td>\n",
" <td>tholus s. m. vitruve nomme \\n tholus coupe d...</td>\n",
" <td>tholus s. m. vitruve nomme \\n tholus coupe d...</td>\n",
" <td>95</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20046</th>\n",
" <td>9</td>\n",
" <td>4327</td>\n",
" <td>MALLIENS, les</td>\n",
" <td>Géographie ancienne</td>\n",
" <td>Geog. anc.</td>\n",
" <td>Jaucourt</td>\n",
" <td>v9-2589-0</td>\n",
" <td>géographie</td>\n",
" <td>Géographie</td>\n",
" <td>MALLIENS, les, (Géog. anc.) en latin Malli ;\\n...</td>\n",
" <td>malliens géog anc latin malli \\n ancien peuple...</td>\n",
" <td>malliens géog anc latin malli \\n ancien peuple...</td>\n",
" <td>71</td>\n",
" </tr>\n",
" <tr>\n",
" <th>35783</th>\n",
" <td>9</td>\n",
" <td>3883</td>\n",
" <td>MAGDOLOS</td>\n",
" <td>Géographie ancienne</td>\n",
" <td>Geog. anc.</td>\n",
" <td>Jaucourt</td>\n",
" <td>v9-2377-0</td>\n",
" <td>géographie</td>\n",
" <td>Géographie</td>\n",
" <td>MAGDOLOS, (Géog. anc.) ville d'Egypte dont\\npa...</td>\n",
" <td>magdolos géog anc ville egypte \\n parlent jéré...</td>\n",
" <td>magdolos géog anc ville egypte \\n parlent jéré...</td>\n",
" <td>50</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-24c25d4d-881e-4c52-8629-da71863f5656')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
" \n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
" </svg>\n",
" </button>\n",
" \n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" flex-wrap:wrap;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-24c25d4d-881e-4c52-8629-da71863f5656 button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-24c25d4d-881e-4c52-8629-da71863f5656');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
" </div>\n",
" "
],
"text/plain": [
" volume ... nb_words\n",
"17253 1 ... 70\n",
"16170 9 ... 36\n",
"12041 16 ... 95\n",
"20046 9 ... 71\n",
"35783 9 ... 50\n",
"\n",
"[5 rows x 13 columns]"
]
},
"metadata": {},
"execution_count": 7
}
]
},
{
"cell_type": "markdown",
"source": [
"## Configuration\n"
],
"metadata": {
"id": "-63bh_cKfN4p"
}
},
{
"cell_type": "code",
"source": [
"columnText = 'contentWithoutClass'\n",
"columnClass = 'ensemble_domaine_enccre'\n",
"\n",
"maxOfInstancePerClass = 10000\n",
"\n",
"batch_size = 64\n",
"validation_split = 0.20\n",
"max_nb_words = 20000 # taille du vocabulaire\n",
"max_sequence_length = 512 # taille max du 'document' \n",
"epochs = 10\n",
"\n",
"#embedding_name = \"fasttext\" \n",
"#embedding_dim = 300 \n",
"\n",
"embedding_name = \"glove.6B.100d\"\n",
"embedding_dim = 100 \n",
"\n",
"path = \"drive/MyDrive/Classification-EDdA/\"\n",
"encoder_filename = \"label_encoder.pkl\"\n",
"tokenizer_filename = \"tokenizer_keras.pkl\""
],
"metadata": {
"id": "nsRuyzYUfOBg"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Preprocessing\n"
],
"metadata": {
"id": "ZDz-Y1LCfQt0"
}
},
{
"cell_type": "code",
"source": [
"if maxOfInstancePerClass != 10000:\n",
" df_train = resample_classes(df_train, columnClass, maxOfInstancePerClass)"
],
"metadata": {
"id": "cP5e7DvRvwxh"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "vGWAgBH87ze8"
},
"source": [
"labels = df_train[columnClass]\n",
"numberOfClasses = labels.nunique()\n",
"\n",
"if os.path.isfile(path+encoder_filename): \n",
" # load existing encoder \n",
" with open(path+encoder_filename, 'rb') as file:\n",
" encoder = pickle.load(file)\n",
"\n",
"else:\n",
" encoder = preprocessing.LabelEncoder()\n",
" encoder.fit(labels)\n",
"\n",
" with open(path+encoder_filename, 'wb') as file:\n",
" pickle.dump(encoder, file)\n",
"\n",
"\n",
"labels = encoder.transform(labels)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"encoder.classes_"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "SME4vvVhW9Sn",
"outputId": "8b577b93-69a5-47e3-ea3b-6a2e1d144914"
},
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"array(['Agriculture - Economie rustique', 'Anatomie', 'Antiquité',\n",
" 'Architecture', 'Arts et métiers', 'Beaux-arts',\n",
" 'Belles-lettres - Poésie', 'Blason', 'Caractères', 'Chasse',\n",
" 'Chimie', 'Commerce', 'Droit - Jurisprudence',\n",
" 'Economie domestique', 'Grammaire', 'Géographie', 'Histoire',\n",
" 'Histoire naturelle', 'Jeu', 'Marine', 'Maréchage - Manège',\n",
" 'Mathématiques', 'Mesure', 'Militaire (Art) - Guerre - Arme',\n",
" 'Minéralogie', 'Monnaie', 'Musique', 'Médailles',\n",
" 'Médecine - Chirurgie', 'Métiers', 'Pharmacie', 'Philosophie',\n",
" 'Physique - [Sciences physico-mathématiques]', 'Politique',\n",
" 'Pêche', 'Religion', 'Spectacle', 'Superstition'], dtype=object)"
]
},
"metadata": {},
"execution_count": 13
}
]
},
{
"cell_type": "code",
"source": [
"labels_index = dict(zip(list(encoder.classes_), encoder.transform(list(encoder.classes_))))"
],
"metadata": {
"id": "nIzWQ2VbW_UO"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"labels_index"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "4e7ggEGiXC_W",
"outputId": "8a22e814-c63c-4e0d-adb9-899c11a1dc9b"
},
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"{'Agriculture - Economie rustique': 0,\n",
" 'Anatomie': 1,\n",
" 'Antiquité': 2,\n",
" 'Architecture': 3,\n",
" 'Arts et métiers': 4,\n",
" 'Beaux-arts': 5,\n",
" 'Belles-lettres - Poésie': 6,\n",
" 'Blason': 7,\n",
" 'Caractères': 8,\n",
" 'Chasse': 9,\n",
" 'Chimie': 10,\n",
" 'Commerce': 11,\n",
" 'Droit - Jurisprudence': 12,\n",
" 'Economie domestique': 13,\n",
" 'Grammaire': 14,\n",
" 'Géographie': 15,\n",
" 'Histoire': 16,\n",
" 'Histoire naturelle': 17,\n",
" 'Jeu': 18,\n",
" 'Marine': 19,\n",
" 'Maréchage - Manège': 20,\n",
" 'Mathématiques': 21,\n",
" 'Mesure': 22,\n",
" 'Militaire (Art) - Guerre - Arme': 23,\n",
" 'Minéralogie': 24,\n",
" 'Monnaie': 25,\n",
" 'Musique': 26,\n",
" 'Médailles': 27,\n",
" 'Médecine - Chirurgie': 28,\n",
" 'Métiers': 29,\n",
" 'Pharmacie': 30,\n",
" 'Philosophie': 31,\n",
" 'Physique - [Sciences physico-mathématiques]': 32,\n",
" 'Politique': 33,\n",
" 'Pêche': 34,\n",
" 'Religion': 35,\n",
" 'Spectacle': 36,\n",
" 'Superstition': 37}"
]
},
"metadata": {},
"execution_count": 15
}
]
},
{
"cell_type": "markdown",
"source": [
"### Loading pre-trained embeddings"
],
"metadata": {
"id": "CoQSwXfNEOOx"
}
},
{
"cell_type": "markdown",
"source": [
"#### FastText"
],
"metadata": {
"id": "PUn9G_LYEYvM"
}
},
{
"cell_type": "code",
"source": [
"# download FastText (prend trop de place pour le laisser sur le drive)\n",
"zip_file_url = \"https://dl.fbaipublicfiles.com/fasttext/vectors-english/crawl-300d-2M.vec.zip\"\n",
"r = requests.get(zip_file_url)\n",
"z = zipfile.ZipFile(io.BytesIO(r.content))\n",
"z.extractall()"
],
"metadata": {
"id": "qKQFmUs0EY5E"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"print('loading word embeddings FastText...')\n",
"\n",
"embeddings_index = {}\n",
"f = codecs.open('crawl-300d-2M.vec', encoding='utf-8')\n",
"\n",
"for line in tqdm(f):\n",
" values = line.rstrip().rsplit(' ')\n",
" word = values[0]\n",
" coefs = np.asarray(values[1:], dtype='float32')\n",
" embeddings_index[word] = coefs\n",
"f.close()\n",
"\n",
"print('found %s word vectors' % len(embeddings_index))"
],
"metadata": {
"id": "x0C0XwhwEY7Z"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"#### GLOVE"
],
"metadata": {
"id": "q81MEgYrEWsj"
}
},
{
"cell_type": "code",
"source": [
"# download Glove\n",
"#zip_file_url = \"https://nlp.stanford.edu/data/glove.6B.zip\"\n",
"#r = requests.get(zip_file_url)\n",
"#z = zipfile.ZipFile(io.BytesIO(r.content))\n",
"#z.extractall()"
],
"metadata": {
"id": "qTHuXs2EEV-M"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"print('loading word embeddings GLOVE...')\n",
"\n",
"embeddings_index = {}\n",
"f = open(path+\"embeddings/\"+embedding_name+\".txt\", encoding='utf-8')\n",
"for line in tqdm(f):\n",
" values = line.split()\n",
" word = values[0]\n",
" coefs = np.asarray(values[1:], dtype='float32')\n",
" embeddings_index[word] = coefs\n",
"f.close()\n",
"\n",
"print('Found %s word vectors.' % len(embeddings_index))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "U0hwICJzEOVL",
"outputId": "be478f5b-6c4a-48ee-e05d-49b2e7fdd285"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"loading word embeddings GLOVE...\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"400000it [00:12, 31570.08it/s]"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Found 400000 word vectors.\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"\n"
]
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "HuUVfklf-dSR"
},
"source": [
"## Training models"
]
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "NTNh6kMTp_eU",
"outputId": "fba63a44-73c5-4928-ad7e-7356b17af073"
},
"source": [
"\n",
"raw_docs_train = df_train[columnText].tolist()\n",
"\n",
"\n",
"print(\"pre-processing train data...\")\n",
"\n",
"if os.path.isfile(path+tokenizer_filename):\n",
" with open(path+tokenizer_filename, 'rb') as file:\n",
" tokenizer = pickle.load(file)\n",
"else:\n",
" tokenizer = Tokenizer(num_words = max_nb_words)\n",
" tokenizer.fit_on_texts(raw_docs_train) \n",
"\n",
" with open(path+tokenizer_filename, 'wb') as file:\n",
" pickle.dump(tokenizer, file)\n",
"\n",
"sequences = tokenizer.texts_to_sequences(raw_docs_train)\n",
"\n",
"word_index = tokenizer.word_index\n",
"print(\"dictionary size: \", len(word_index))\n",
"\n",
"#pad sequences\n",
"data = sequence.pad_sequences(sequences, maxlen=max_sequence_length)\n",
"\n",
"print('Shape of data tensor:', data.shape)\n",
"print('Shape of label tensor:', labels.shape)\n",
"#print(labels)"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"pre-processing train data...\n",
"dictionary size: 190508\n",
"Shape of data tensor: (46807, 512)\n",
"Shape of label tensor: (46807,)\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# split the data into a training set and a validation set\n",
"\n",
"indices = np.arange(data.shape[0])\n",
"np.random.shuffle(indices)\n",
"data = data[indices]\n",
"labels = labels[indices]\n",
"\n",
"nb_validation_samples = int(validation_split * data.shape[0])\n",
"\n",
"x_train = data[:-nb_validation_samples]\n",
"y_train = labels[:-nb_validation_samples]\n",
"x_val = data[-nb_validation_samples:]\n",
"y_val = labels[-nb_validation_samples:]\n"
],
"metadata": {
"id": "sHYJ4P-YDfFb"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "wGjQI0YgpQAS",
"outputId": "ab38f7a8-8c10-471f-88e6-10a62ae1776c"
},
"source": [
"#embedding matrix\n",
"\n",
"print('preparing embedding matrix...')\n",
"\n",
"embedding_matrix = np.zeros((len(word_index)+1, embedding_dim))\n",
"\n",
"for word, i in word_index.items():\n",
" embedding_vector = embeddings_index.get(word)\n",
" if embedding_vector is not None : \n",
" embedding_matrix[i] = embedding_vector\n"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"preparing embedding matrix...\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"\n",
"#filter_sizes = [2, 3, 5]\n",
"#drop = 0.5\n",
"\n",
"embedding_layer = Embedding(len(word_index)+1, embedding_dim, input_length = max_sequence_length,\n",
" weights=[embedding_matrix], trainable=False)\n",
"inputs = Input(shape=(max_sequence_length), dtype='int32')\n",
"embedding = embedding_layer(inputs)\n",
"\n",
"print(embedding.shape)\n",
"#reshape = Reshape((max_sequence_length, embedding_dim, 1))(embedding)\n",
"#print(reshape.shape)\n",
"\n",
"# architecture testée par Khaled\n",
"\n",
"conv_0 = Conv1D(64, 5, activation='relu')(embedding)\n",
"#conv_1 = Conv1D(128, 5, activation='relu')(embedding)\n",
"#conv_2 = Conv1D(128, 5, activation='relu')(embedding)\n",
"\n",
"maxpool_0 = MaxPool1D(pool_size=(max_sequence_length - 5 + 1))(conv_0)\n",
"#maxpool_1 = MaxPool1D(5)(conv_1)\n",
"#maxpool_2 = MaxPool1D(35)(conv_2)\n",
"\n",
"#concatenated_tensor = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2])\n",
"flatten = Flatten()(maxpool_0)\n",
"#dropout = Dropout(drop)(flatten)\n",
"output = Dense(len(labels_index), activation='softmax')(flatten)\n",
"\n",
"# this creates a model that includes\n",
"model = Model(inputs=inputs, outputs=output)\n",
"\n",
"checkpoint = ModelCheckpoint('weights_cnn_sentece.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='auto')\n",
"#adam = Adam(lr=1e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)\n",
"\n",
"model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['acc'])\n",
"model.summary()"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "OUphqlYJCC9n",
"outputId": "7b8a479d-ba34-4ea7-b714-107b9c7166b3"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"(None, 512, 100)\n",
"Model: \"model\"\n",
"_________________________________________________________________\n",
" Layer (type) Output Shape Param # \n",
"=================================================================\n",
" input_1 (InputLayer) [(None, 512)] 0 \n",
" \n",
" embedding (Embedding) (None, 512, 100) 19050900 \n",
" \n",
" conv1d (Conv1D) (None, 508, 64) 32064 \n",
" \n",
" max_pooling1d (MaxPooling1D (None, 1, 64) 0 \n",
" ) \n",
" \n",
" flatten (Flatten) (None, 64) 0 \n",
" \n",
" dense (Dense) (None, 38) 2470 \n",
" \n",
"=================================================================\n",
"Total params: 19,085,434\n",
"Trainable params: 34,534\n",
"Non-trainable params: 19,050,900\n",
"_________________________________________________________________\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"history = model.fit(x_train, y_train, \n",
" batch_size=batch_size, \n",
" epochs=epochs, \n",
" verbose=1,\n",
" callbacks=[checkpoint],\n",
" validation_data=(x_val, y_val))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "3aUBdLdNCEGK",
"outputId": "591e58dc-c403-445a-dab6-3d902f0f6465"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Epoch 1/10\n",
"585/586 [============================>.] - ETA: 0s - loss: 2.0486 - acc: 0.4831\n",
"Epoch 1: val_acc improved from -inf to 0.56832, saving model to weights_cnn_sentece.hdf5\n",
"586/586 [==============================] - 93s 157ms/step - loss: 2.0484 - acc: 0.4831 - val_loss: 1.6547 - val_acc: 0.5683\n",
"Epoch 2/10\n",
"585/586 [============================>.] - ETA: 0s - loss: 1.4558 - acc: 0.6155\n",
"Epoch 2: val_acc improved from 0.56832 to 0.61949, saving model to weights_cnn_sentece.hdf5\n",
"586/586 [==============================] - 85s 145ms/step - loss: 1.4557 - acc: 0.6156 - val_loss: 1.4356 - val_acc: 0.6195\n",
"Epoch 3/10\n",
"585/586 [============================>.] - ETA: 0s - loss: 1.2437 - acc: 0.6631\n",
"Epoch 3: val_acc improved from 0.61949 to 0.63829, saving model to weights_cnn_sentece.hdf5\n",
"586/586 [==============================] - 84s 143ms/step - loss: 1.2439 - acc: 0.6631 - val_loss: 1.3358 - val_acc: 0.6383\n",
"Epoch 4/10\n",
"585/586 [============================>.] - ETA: 0s - loss: 1.1175 - acc: 0.6942\n",
"Epoch 4: val_acc improved from 0.63829 to 0.65111, saving model to weights_cnn_sentece.hdf5\n",
"586/586 [==============================] - 84s 143ms/step - loss: 1.1176 - acc: 0.6941 - val_loss: 1.2895 - val_acc: 0.6511\n",
"Epoch 5/10\n",
"585/586 [============================>.] - ETA: 0s - loss: 1.0243 - acc: 0.7172\n",
"Epoch 5: val_acc improved from 0.65111 to 0.65356, saving model to weights_cnn_sentece.hdf5\n",
"586/586 [==============================] - 84s 143ms/step - loss: 1.0242 - acc: 0.7172 - val_loss: 1.2751 - val_acc: 0.6536\n",
"Epoch 6/10\n",
"585/586 [============================>.] - ETA: 0s - loss: 0.9492 - acc: 0.7371\n",
"Epoch 6: val_acc improved from 0.65356 to 0.65987, saving model to weights_cnn_sentece.hdf5\n",
"586/586 [==============================] - 92s 158ms/step - loss: 0.9491 - acc: 0.7371 - val_loss: 1.2598 - val_acc: 0.6599\n",
"Epoch 7/10\n",
"585/586 [============================>.] - ETA: 0s - loss: 0.8892 - acc: 0.7536\n",
"Epoch 7: val_acc did not improve from 0.65987\n",
"586/586 [==============================] - 86s 147ms/step - loss: 0.8892 - acc: 0.7536 - val_loss: 1.2598 - val_acc: 0.6557\n",
"Epoch 8/10\n",
"585/586 [============================>.] - ETA: 0s - loss: 0.8387 - acc: 0.7659\n",
"Epoch 8: val_acc improved from 0.65987 to 0.66179, saving model to weights_cnn_sentece.hdf5\n",
"586/586 [==============================] - 85s 145ms/step - loss: 0.8387 - acc: 0.7659 - val_loss: 1.2452 - val_acc: 0.6618\n",
"Epoch 9/10\n",
"585/586 [============================>.] - ETA: 0s - loss: 0.7950 - acc: 0.7780\n",
"Epoch 9: val_acc improved from 0.66179 to 0.66275, saving model to weights_cnn_sentece.hdf5\n",
"586/586 [==============================] - 83s 142ms/step - loss: 0.7950 - acc: 0.7780 - val_loss: 1.2593 - val_acc: 0.6627\n",
"Epoch 10/10\n",
"585/586 [============================>.] - ETA: 0s - loss: 0.7575 - acc: 0.7873\n",
"Epoch 10: val_acc improved from 0.66275 to 0.66286, saving model to weights_cnn_sentece.hdf5\n",
"586/586 [==============================] - 83s 141ms/step - loss: 0.7575 - acc: 0.7873 - val_loss: 1.2646 - val_acc: 0.6629\n"
]
}
]
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "ZcYbQsQEJKLq"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"\n",
"plt.plot(history.history['acc'])\n",
"plt.plot(history.history['val_acc'])\n",
"plt.title('model accuracy')\n",
"plt.ylabel('accuracy')\n",
"plt.xlabel('epoch')\n",
"plt.legend(['train', 'validation'], loc='lower right')\n",
"plt.show()\n",
"\n",
"# summarize history for loss\n",
"plt.plot(history.history['loss'])\n",
"plt.plot(history.history['val_loss'])\n",
"plt.title('model loss')\n",
"plt.ylabel('loss')\n",
"plt.xlabel('epoch')\n",
"plt.legend(['train', 'validation'], loc='upper right')\n",
"plt.show()\n",
"\n"
],
"metadata": {
"id": "Job-3uMvJKN_",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 573
},
"outputId": "612f25b7-e9ad-43c6-e084-041e2a5f3b64"
},
"execution_count": null,
"outputs": [
{
"output_type": "display_data",
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9b34/9c7GyEkJCEBAkkgQYGwCEQCakGrdUMrUFfAasFWab1al9vl6r391aXt/Xm719YuVmltVRZRFC2VouJCXUjCTtghIQkkhCQkISFkmff3j3OAIQ4wgUwmy/v5eMwjZ/vMvGeU8z6f5XyOqCrGGGNMSyHBDsAYY0zHZAnCGGOMT5YgjDHG+GQJwhhjjE+WIIwxxvhkCcIYY4xPliCMAUTkryLyYz+PzReRqwIdkzHBZgnCGGOMT5YgjOlCRCQs2DGYrsMShOk03Kad74nIBhGpFZHnRaS/iPxTRGpE5B0Rifc6fpqIbBaRQyLyvoiM8NqXKSJr3HILgcgWn3WDiKxzy34sImP8jPHLIrJWRKpFpFBEHm+xf7L7fofc/XPc7T1F5BciUiAiVSKyyt12uYgU+fgdrnKXHxeRxSLyoohUA3NEZKKIfOJ+xn4R+Z2IRHiVHyUiK0SkQkRKReS/RSRJROpEJMHruAtFpExEwv357qbrsQRhOpubgauBYcBU4J/AfwN9cf5/fgBARIYB84GH3H3LgDdFJMI9Wb4O/B3oA7zivi9u2UxgHvBNIAH4E7BURHr4EV8t8DUgDvgycK+IfMV938FuvL91YxoHrHPL/RwYD3zBjen7gMfP32Q6sNj9zJeAZuBhIBG4BLgS+A83hhjgHeBtYCBwPvCuqpYA7wO3eb3vncACVW30Mw7TxViCMJ3Nb1W1VFWLgY+Az1R1rarWA0uATPe4GcA/VHWFe4L7OdAT5wR8MRAO/FpVG1V1MZDt9RlzgT+p6meq2qyqLwBH3XKnparvq+pGVfWo6gacJPVFd/ftwDuqOt/93HJVXSciIcDXgQdVtdj9zI9V9aifv8knqvq6+5lHVDVXVT9V1SZVzcdJcMdiuAEoUdVfqGq9qtao6mfuvheAOwBEJBSYhZNETTdlCcJ0NqVey0d8rEe7ywOBgmM7VNUDFALJ7r5iPXmmygKv5cHAd9wmmkMicghIdcudlohcJCIr3aaZKuBbOFfyuO+xy0exRJwmLl/7/FHYIoZhIvKWiJS4zU7/60cMAG8AI0UkHaeWVqWqq88yJtMFWIIwXdU+nBM9ACIiOCfHYmA/kOxuO2aQ13Ih8BNVjfN6RanqfD8+92VgKZCqqrHAH4Fjn1MInOejzEGg/hT7aoEor+8RitM85a3llMx/ALYCQ1W1N04TnHcMQ3wF7tbCFuHUIu7Eag/dniUI01UtAr4sIle6nazfwWkm+hj4BGgCHhCRcBG5CZjoVfbPwLfc2oCISC+38znGj8+NASpUtV5EJuI0Kx3zEnCViNwmImEikiAi49zazTzglyIyUERCReQSt89jOxDpfn448APgTH0hMUA1cFhEMoB7vfa9BQwQkYdEpIeIxIjIRV77/wbMAaZhCaLbswRhuiRV3YZzJfxbnCv0qcBUVW1Q1QbgJpwTYQVOf8VrXmVzgHuA3wGVwE73WH/8B/CkiNQAP8RJVMfedy9wPU6yqsDpoB7r7v4usBGnL6QC+D8gRFWr3Pd8Dqf2UwucNKrJh+/iJKYanGS30CuGGpzmo6lACbADuMJr/79xOsfXqKp3s5vphsQeGGSM8SYi7wEvq+pzwY7FBJclCGPMcSIyAViB04dSE+x4THBZE5MxBgAReQHnHomHLDkYsBqEMcaYUwhoDUJEpojINhHZKSKP+Ng/yB0zvlac6ROu99r3qFtum4hcG8g4jTHGfF7AahDueO3tOCMminBGZ8xS1TyvY54F1qrqH0RkJLBMVdPc5fk4Qw8H4lR7h6lq86k+LzExUdPS0gLyXYwxpqvKzc09qKot760BIJAzP04EdqrqbgARWYAzZ0ye1zEK9HaXY3FubsI9boE71cAeEdnpvt8np/qwtLQ0cnJy2vYbGGNMFycipxzOHMgmpmROngKgyN3m7XHgDne2ymXAt1tRFhGZKyI5IpJTVlbWVnEbY4wh+KOYZgF/VdUUnBuI/u5OXOYXVX1WVbNUNatvX581JGOMMWcpkE1MxThz3xyT4m7z9g1gCoCqfiIikTiTivlT1hhjTAAFsgaRDQwVkXR3/v2ZOJOYeduLM1c94jzMJRIoc4+b6c4Vkw4MBWxWSWOMaUcBq0GoapOI3A8sB0KBeaq6WUSeBHJUdSnOnDR/FpGHcTqs57hTMG8WkUU4HdpNwH2nG8FkjDGm7XWZG+WysrLURjEZY0zriEiuqmb52hfsTmpjjDEdVCA7qY0xxgTIoboGtpbUsHV/NeFhIXz1osFnLtRKliCMMaYDa2z2sOdgLVv2Vx9PCFtLathfVX/8mAsHxVmCMMaYrqys5ihbS6rZur+GLe7fnQcO09DsASA8VDivbzQXD0kgIymGjAG9GZEUQ9+YMz1k8OxYgjDGmHZW39jMzgOHT9QK3GRQXttw/Jj+vXuQkdSbS4clMiKpNxkDYhiSGE1EWPt1HVuCMMaYAFFV9lXVH28WOpYQ9hyspdnjjCDtERbC8KQYrhzRjww3EWQk9aZPr4ggR28Jwhhj2kTt0Sa2ldawdX/NSc1ENfVNx49J7dOTjKTeXD86iYwBvclIimFwQi9CQySIkZ+aJQhjjGmlqrpG1uytZF3hIScZlNRQUF53fH90jzAykmKYPm4gGUm9GTEghmH9Y4iJDA9i1K1nCcIYY05DVSmqPEJOQQU5+ZXk5FeyrdR5ImuIQFpiL0YPjOWWC1OO1wpS4nsi0jFrBa1hCcIYY7w0NXvYWlJDTn4F2QWV5OZXUlLtDCmN6RHGhYPjuWHMALLS+jA2NZaoiK57Gu2638wYY/xQe7SJdYWHyM6vILegkjUFldQ2OFO/Jcf1ZGJ6HyakxTN+cB+GJ8V02P6CQLAEYYzpVg5U15OdX3m8yShvfzXNHkUEMpJ6c/P4FMYPjicrrQ/JcT2DHW5QWYIwxnRZHo+yq+ywkxDyK8gpqGRvhdOZHBkeQmZqPP9x+XlkpfUhc1AcvTtZJ3KgWYIwxnQZ9Y3NbCyucjuTnYRQdaQRgMToHmQNjudrlwxmQlofRg7sTXiozVd6OpYgjDGdVmVtA7kFlWS7zUUbi6qOT0txXt9eXDc6ifGD45mQ1ofBCVFdYmRRe7IEYYzpFFSVwoojZOdXkFNQQXZ+JTsPHAacOYrGpMRx16Q0stL6MH5wfIe4E7mzswRhjOmQmj3K1pJqsvc4w01z8isorT4KQO/IMLLS+nDThclkDe7DmJRYIsNDgxxx12MJwhjTIdQ3NrOu8JBz/0G+M9y05qgzTUVyXE8uHpLAhLQ+TEjrw9B+0YR0o+GmwWIJwhgTFIfqGsjJd/oPsvdUsLG4isZmZwK74f1jmDZuIBPT+9hw0yCyBGGMCThVpfiQ039wbMjp9tKT+w++MXmIe0NaPHFR1n/QEViCMMa0OY9H2VZac7y5KDu/4vgT0I5NVzF9XDJZg+MZmxpn/QcdlCUIY8w5O3b/QXa+01yUW1BJtTvNdf/ePY73HUxI637TVXRmliCMMa1WdaSRNQWVrM6vICe/gvVFVTQ0OfcfnN8vmi+PGUDW4D5MTO/TZWY27Y4CmiBEZArwGyAUeE5Vn2qx/1fAFe5qFNBPVePcfc3ARnffXlWdFshYjTGn1tDk4ZPd5by7pZTVeyrYVlqDKoSFCKOTY5nt3p08fnA8CdGBeT6yaX8BSxAiEgo8A1wNFAHZIrJUVfOOHaOqD3sd/20g0+stjqjquEDFZ4w5vdqjTXywvYzlm0t4b+sBauqb6BkeSlZaPNeNHsCE9HjGpcZ16emuu7tA/pedCOxU1d0AIrIAmA7kneL4WcBjAYzHGHMGFbUNvJNXyvLNJXy08yANTR7io8KZMiqJa0clMXloonUodyOBTBDJQKHXehFwka8DRWQwkA6857U5UkRygCbgKVV9PVCBGtOdFVXW8a/NTlLIzq/Ao86NabdPHMS1o5KYkBZPmE1q1y11lLrhTGCxqjZ7bRusqsUiMgR4T0Q2quou70IiMheYCzBo0KD2i9aYTkxV2XHgMMs3lbA8r4RNxdUADOsfzX1XnM81I5MYndzbOpZNQBNEMZDqtZ7ibvNlJnCf9wZVLXb/7haR93H6J3a1OOZZ4FmArKwsbZOojemCPB5lbeEh/pVXwr82l7LnYC0AmYPieOS6DK4dlUR6Yq8gR2k6mkAmiGxgqIik4ySGmcDtLQ8SkQwgHvjEa1s8UKeqR0UkEZgE/DSAsRrT5TQ0efh0dznLN5ewIq+UAzVHCQsRLjkvga9PTueakf3p3zsy2GGaDixgCUJVm0TkfmA5zjDXeaq6WUSeBHJUdal76Exggap61wBGAH8SEQ8QgtMHcarObWOMq66hiQ+2OSOP3vUaefTFYX25dnR/vjS8P7FR9tQ04x85+bzceWVlZWlOTk6wwzCm3VXWNvDOllKWby7lox1lHG3yEBcVzlUj+nPtqCQutZFH5jREJFdVs3zt6yid1MaYVig+dIR/bS5xRx5V0uxRBsZGMmviIK4Z1Z+JaX1s5JE5Z5YgjOkEVJWdBw6zfHMJyzeXsrG4CnCmtfjWF4dw7agkLkiOtZFHpk1ZgjCmAztQXc9ra4tZnFt0/PGa41Lj+K8pGVwzqj/n9Y0OcoSmK7MEYUwH09js4b2tB3glp5CV28po9igT0uL50fRRXD0yiaRYG3lk2oclCGM6iB2lNSzKKWTJ2mIOHm6gX0wP5l42hFvHpzDEagomCCxBGBNENfWNvLVhP4tyClm79xBhIcKVI/oxY0Iqlw3tax3NJqgsQRjTzlSVz/ZUsCinkGUb91Pf6GFov2h+8OURfCUzmUSbLtt0EJYgjGkn+6uO8GpuEa/kFlFQXkdMjzBuujCF27JSGZtiI5BMx2MJwpgAOtrUzLtbDrAwu5CPdpThUbhkSAIPXTWUKaMG0DPCbmAzHZclCGMCYMv+ahblFPL62mIq6xoZEBvJfVeczy3jUxicYJPimc7BEoQxbaSqrpGl64tZlFPExuIqIkJDuHpUf27LSmXy+YmEhlgTkulcLEEYcw48HuXjXeUsyink7c0lNDR5GDGgN49PHcn0ccnE94oIdojGnDVLEMachaLKOhbnFvFKThHFh44Q2zOcWRNSuTUrldHJscEOz5g2YQnCGD/VNzazfHMJr+QU8e9dBwGYfH4i/3VdBteM7G8zppouxxKEMaehqmwqdjqc31hXTHV9EynxPXnoymHcPD6ZlPioYIdoTMBYgjDGh4YmD29t2Me8f+9hU3E1PcJCuG50ErdlpXLxkARCrMPZdAOWIIzxUlnbwMur9/LCx/kcqDnK+f2i+dH0UUwbl0xsT3sSm+leLEEYA+wqO8y8VXt4dU0R9Y0eLh2ayE9vGcMXh/W1O5xNt2UJwnRbqsonu8p5btUe3tt6gIiwEG4cl8zXJ6czPCkm2OEZE3SWIEy3c7SpmTfX7+e5j3aztaSGxOgIHrpqKHdcPNgmyjPGiyUI021U1Dbw0qcF/O3TAspqjjK8fww/vXkM08YNtCGqxvhgCcJ0eTsP1PD8qnxeW1PE0SYPlw/vyzcmpzP5/ETrXzDmNCxBmC5JVVm18yDPr9rD+9vK6BEWwk0XpvCNyWmc38/6F4zxhyUI06XUNzazdN0+nl+1h22lNSRG9+A7Vw/j9osGkWD9C8a0SkAThIhMAX4DhALPqepTLfb/CrjCXY0C+qlqnLtvNvADd9+PVfWFQMZqOreDh4/y4qcFvPhpAQcPN5CRFMPPbx3L1LED6BFm/QvGnI2AJQgRCQWeAa4GioBsEVmqqnnHjlHVh72O/zaQ6S73AR4DsgAFct2ylYGK13RO20treP6jPSxZV0xDk4cvZfTj7snpXHJegvUvGHOOAlmDmAjsVNXdACKyAJgO5J3i+Fk4SQHgWmCFqla4ZVcAU4D5AYzXdBKqygfby3h+1R4+2nGQyPAQbh2fwtcnp3Ne3+hgh2dMlxHIBJEMFHqtFwEX+TpQRAYD6cB7pymb7KPcXGAuwKBBg849YtOh1Tc2s2RtMfNW7WHHgcP0i+nB964dzu0TB9lzF4wJgI7SST0TWKyqza0ppKrPAs8CZGVlaSACM8F3oKaeFz8p4MXP9lJR28Cogb351YyxfPmCgUSEhQQ7PGO6rEAmiGIg1Ws9xd3my0zgvhZlL29R9v02jM10Alv2V/P8qj0sXbePRo+HKzP6843J6Vw8pI/1LxjTDgKZILKBoSKSjnPCnwnc3vIgEckA4oFPvDYvB/5XROLd9WuARwMYq+lA9pbX8aN/5LEir5Se4aHMnJjKXZPSSU/sFezQjOlWApYgVLVJRO7HOdmHAvNUdbOIPAnkqOpS99CZwAJVVa+yFSLyI5wkA/DksQ5r03XVNzbz+/d38ccPdhEWInzn6mHceclg4qKsf8GYYBCv83KnlpWVpTk5OcEOw5wFVWVFXilPvpVHUeURpo4dyP9cP4Kk2Mhgh2ZMlyciuaqa5WtfR+mkNt3UnoO1PPHmZt7fVsaw/tHMv+diLjkvIdhhGWOwBGGCpK6hiWdW7uTPH+4hIiyEH3x5BLO/kEZ4qI1KMqajsARh2pWq8s9NJfz4rTz2VdVzU2Yyj1yfQb8Ya04ypqOxBGHazc4DNTy+NI9VOw+SkRTDb2ZlMiGtT7DDMsacgiUIE3CHjzbx23d38PyqPfSMCOWJaaP46kWDCLPmJGM6NEsQJmBUlaXr9/G/y7ZQWn2U27JS+P6UDHuspzGdhCUIExDbSmp4bOkmPt1dwejk3vzhjvFcOCj+zAWNMR2GJQjTpqrrG/n1ih288Ek+MZFh/OTG0cycMIjQEJsaw5jOxhKEaROqypK1xfzvsq2U1x5l1sRBfO+a4TbLqjGdmCUIc84276visTc2k1NQybjUOObNyWJMSlywwzLGnCNLEOasVdU18osV23jx0wLioiL46c1juGV8CiHWnGRMl2AJwrSax6Mszi3i/97eSmVdA3dePJj/vHo4sVHhwQ7NGNOGLEGYVtlQdIgfvrGZdYWHmJAWzxPTLmLkwN7BDssYEwCWIIxfKmsb+Nm/tjF/9V4SevXgl7eN5cbMZHtwjzFdmCUIc1rNHmVB9l5+tnwbNfVNfH1SOg9eNZTekdacZExXZwnCnNKavZU89sZmNhZXcVF6H56cPprhSTHBDssY0078ShAi8hrwPPBPVfUENiQTbOWHj/J/b29lUU4R/Xv34OlZmUwdM8Cak4zpZvytQfweuAt4WkReAf6iqtsCF5YJlrc3lfD9xeupa2jmm18cwgNfGkqvHlbRNB2cxwOeRmhudP56mr2Wm6C5yWt/cyuObXK3+Ti2uRGOXS8ffzKnnrzccl+brPvY1ycdvvj9s/75TsWvf/mq+g7wjojEArPc5ULgz8CLqtrY5pGZdvfSZwX84PVNjE2J4+e3juX8ftHBDsl0V00NUF0MVUXOq7roxHJVEdSUnHyipp0fnRwSDqHhIKFwvGYtXn+OLbfc14br3sv1VefybU7J70tDEUkA7gDuBNYCLwGTgdnA5YEIzrQPVeV37+3kFyu286WMfjxz+4X0jAgNdljdV301VBXCoUL3717nZBkaAVEJzqtXorvs/u2VAD1iIaQTTKGuCnUVznc7ftIvPDkBHC7lcyf9qESITYGE8yFtMoRFQkiYc6IOCfNadtdDw7yWw/071uf+0BPLoeEgIV4n567N3z6IJcBw4O/AVFXd7+5aKCI5gQrOBJ7Hozzx5mZe+KSAmzKT+b9bxthjPwNJFWoPQtVerwTg/Xfv568GQyOg90Cn6aPuIDTV+35vCW2RQPp4JZDEE/u818MCMPV6Y/3JV/++EkDTkZPLhEU6J//YFBh6FcSmnljvnQKxyRDes+1jNaflbw3iaVVd6WuHqma1YTymHTU0efjOK+t5c/0+7rk0nUevG2HTZJyr5iao2X/yCf+kBODj5BgRA3Gpzklx0EXO37hUiB3k/O3V7+SaQUMt1JU7iaauwkkax9fLT7wObHG2HanklE0wETFOIjmpRtJy3Svh9Ih1Pq/lCd/7VXvg858TneSc7PuPhGHXnjj5x6Y43zcqodtclXcm/iaIkSKyVlUPAYhIPDBLVX8fuNBMINUebeJbL+by0Y6DPHpdBt/84nnBDqlzaKx3T4SnqAFUF4M2n1wmKtE50fcb4Z4cU08khLhUiIxr3ckxopfzihvk3/GeZidJnJREjiWV8hPrh0uhNO/0tRRfwnudONknXXDy1X9silP7CURNxQScvwniHlV95tiKqlaKyD04o5tOSUSmAL8BQoHnVPUpH8fcBjyOc4mzXlVvd7c3Axvdw/aq6jQ/YzVnUFHbwF1/zWZj0SF+essYbstKDXZI7cfjgcZa5yq8oRYaDp+8fPTw57cfLj2RAFpeHUsIxAx0TvSDLj75xB87yDlBRkQF57seExLq1AB6JULf4f6V+VwtxU0i9VXQqy/0Tj6RAHrG29V/F+VvgggVEVF1xlSJSChw2on+3WOeAa4GioBsEVmqqnlexwwFHgUmuUmnn9dbHFHVca34LsYPxYeOcOfzn1FceYQ/3ZnF1SP7BzukU2tuOv3JvKHW9wn9pOVj6+62xjr/P19CoUf0iRrAsGudq3bvGkDvgU7HZVfT2lqK6ZL8TRBv43RI/8ld/6a77XQmAjtVdTeAiCwApgN5XsfcAzyjqpUAquqj8dK0lR2lNdz5/GpqG5r4+zcuYmJ6n2CHdLLKAij4N+T/GwpWQWW+/2VDe7gntegTJ7eIXk7btvd6RLRz0v/csT6WQyPsyth0a/4miP/CSQr3uusrgOfOUCYZKPRaLwIuanHMMAAR+TdOM9Tjqnos8US6I6SagKdU9fWWHyAic4G5AIMG2ZXO6eQWVPL1v2YTERbCom9ewogBQZ6BVRUqdnslhH87zTjgNFkMngRjZ0GPGN8n/pYn9K54FW9MkPl7o5wH+IP7auvPH4pzH0UK8KGIXOB2hg9W1WIRGQK8JyIbVXVXi7ieBZ4FyMrKauc7ZTqPldsOcO+LuST1juTv37iI1D5BaBNXhYM7nJpB/r+h4GOo2efsi0qEtEnwhQecv31HdI7x/MZ0cf7eBzEU+P+BkUDkse2qOuQ0xYoB797PFHebtyLgM/dO7D0ish0nYWSrarH7GbtF5H0gE9iFaZUla4v43isbyBgQw1/vmkhidDuNJlGFsq2Qv+pELeFYB290f6eGkDbZeSUOs6YcYzogf5uY/gI8BvwKuAJnXqYzXeJlA0NFJB0nMcwEbm9xzOs4U3f8RUQScZqcdrvDaOtU9ai7fRLwUz9jNa7nV+3hR2/lccmQBJ792nhiAjlFt8cDBzaf6D8o+NgZ+QLOiJchlzu1g8GTIeE8SwjGdAL+JoieqvquO5KpAHhcRHKBH56qgKo2icj9wHKc/oV5qrpZRJ4EclR1qbvvGhHJA5qB76lquYh8AfiTiHhwEtFT3qOfzOmpKj9dvo0/vL+L60Yn8asZ44gMb+OpMzzNULLhRP9BwcdQf8jZFzcIhl7rJoRJEJ9mCcGYTsjfBHFUREKAHe5Jvxg440xuqroMWNZi2w+9lhX4T/flfczHwAV+xma8NDV7+O8lG1mUU8TtFw3iR9NHE9oWd0c3N8L+9SeajPZ+CkernX19hsCIqU5z0eBJzhBQY0yn52+CeBCIAh4AfoTTzDQ7UEGZs1Pf2My3569lRV4pD1w5lIevGnr2z3BoaoB9a7wSwmfOPQkACUNh9E1Oc1HaJOdeAGNMl3PGBOHe8DZDVb8LHMbpfzAdTNWRRu75Ww7Z+RU8MW0Us7+Q1vo3qT0Im16DrW9B4eoTcwb1HQHjZjm1g8GTIKYD31xnjGkzZ0wQqtosIpPbIxhzdg5U1/O1eavZVXaY38zMZNrYVlzRNx6BbctgwyLY+Y7zcJS+GTB+tpsQvuBM0WCM6Xb8bWJaKyJLgVeA2mMbVfW1gERl/FZQXsudz6/m4OGjPD97ApcN63vmQh6PM9Jo/ULIewMaaiBmAFz8HzBmBiSNDnzgxpgOz98EEQmUA1/y2qaAJYgg2lRcxZy/rKbZo7x8z8WMS407fYHSPNiwEDa+4sw6GhENI6fDmNsg7VJnUjdjjHH5eye19Tt0MJ/sKueev+UQ2zOcF74+8dSPB60pcRLChoVQstGZgO78q+DqJ2H49cGfadQY02H5eyf1X/DxxBFV/XqbR2TO6O1N+3lg/joGJ0Txt29MZEBsiydtHT0MW950ksKeD5wHqw+8EK77KYy6CaL9aIYyxnR7/jYxveW1HAncCOxr+3DMmcxfvZf/WbKRcalxzJszgbgod9b15ibY/T5sWABb/+FMax03GC79rtOElDg0qHEbYzoff5uYXvVeF5H5wKqARGR8UlWeWbmTn/9rO5cP78vvv3ohUeGhsG+tMwJp42JnrqPIOKejeexMSL3I7mA2xpw1f2sQLQ0F+p3xKNMmPB7lybfy+OvH+dyYmcxPr4oj/NNfO4nh4DbnuQXDrnUSw9Br7PGOxpg24W8fRA0n90GU4DwjwgRYQ5OH776ynvfX7+C3w3dzQ+3vkN9+7OwcdAnc8GtnJFJUB3v4jzGm0/O3iSkm0IGYz6utq+PZeX9iSukyftlzHWEFDZBwPlzxAxhzqzMJnjHGBIi/NYgbgfdUtcpdjwMu9/WUN3OOVKFwNfVr5tO8fjEPaw31PfsQlvl1pwlpYKb1Kxhj2oW/fRCPqeqSYyuqekhEHsN5noNpCw118PHTsH6++yzmCD7wZJF82RwuvOIme6SmMabd+ZsgfD0c6Gw7uE1LDbXw8gzIX0Vd8iR+UTOVfzSN5zdzLuPCIQnBjs4Y0035e5LPEZFfAs+46/cBuYEJqZs5WgMv3QaFn7Lni7/mxo8GEh4awgtzJzJyYO9gR2eM6cb8fTL8t4EGYCGwAKjHSRLmXNRXwwErU2wAABb4SURBVIs3Q+FnlFz9DNe/l0Rsz3Be/dYXLDkYY4LO31FMtcAjAY6le6mvgr/fBPvXwa1/4Q87zqNZC1n0zUvo3zsy2NEZY4x/NQgRWeGOXDq2Hi8iywMXVhd3pBL+Nt15hOdtf6N+6A0sWVvMdaOTLDkYYzoMf5uYElX10LEVVa3E7qQ+O3UV8MI0KN0MM16EjC/zz037qa5vYsYEe5azMabj8DdBeERk0LEVEUnDx+yu5gxqy53kULYNZr4Mw6cAMH91IWkJUVxiI5aMMR2Iv6OY/gdYJSIfAAJcCswNWFRd0eEyp1mpYhfMmg/nXwnA7rLDrN5TwfenDEfsBjhjTAfibyf12yKShZMU1uLcIHckkIF1KTWl8LdpUFkAty+EIZcf37Uwu5CwEOGW8SlBC88YY3zxd6qNu4EHgRRgHXAx8AknP4LU+FK9H16YCtX74I7FkDb5+K6GJg+vriniyhH96BdjndPGmI7F3z6IB4EJQIGqXgFkAodOXwREZIqIbBORnSLic5isiNwmInkisllEXvbaPltEdriv2X7G2bFUFcNfvww1++GOV09KDgDvbinl4OEGZk4YdIo3MMaY4PG3D6JeVetFBBHpoapbRWT46QqISCjOnddXA0VAtogsVdU8r2OGAo8Ck1S1UkT6udv7AI8BWTid4blu2cpWf8NgOVQIL9zgdEzfuQRSJ37ukPnZhQyIjeSyYfYIUGNMx+NvDaLIvQ/idWCFiLwBFJyhzERgp6ruVtUGnDuwp7c45h7gmWMnflU94G6/FlihqhXuvhXAFD9jDb7KAvjr9VBXCV973WdyKKqs46MdZdyalUpoiHVOG2M6Hn87qW90Fx8XkZVALPD2GYolA4Ve60XARS2OGQYgIv8GQoHHVfXtU5RNbvkBIjIXdzTVoEEdpJmmYo/T53C0xkkOyRf6PGxRThEAt2VZ57QxpmNq9YysqvpBG3/+UOBynA7wD0XkglbE8izwLEBWVlbw78so3+Ukh8Y6mL0UBoz1eVizR3klp5DLhvYlJT6qnYM0xhj/+NvEdDaKAe9bg1Pcbd6KgKWq2qiqe4DtOAnDn7Idy8EdTod0Uz3MfvOUyQHgw+1l7K+qZ6bdOW2M6cACmSCygaEiki4iEcBMYGmLY17HqT0gIok4TU67geXANe6cT/HANe62junAVic5eJpg9luQdPpK0PzVe0mMjuDKEf3bKUBjjGm9gD30R1WbROR+nBN7KDBPVTeLyJNAjqou5UQiyAOage+pajmAiPwIJ8kAPKmqFYGK9ZyU5jk3wUkIzPkH9D3t4C4O1NTz7tYD3D05nYiwQOZnY4w5NwF9KpyqLgOWtdj2Q69lBf7TfbUsOw+YF8j4zlnJJic5hEY4zUqJQ89YZHFuEc0etYn5jDEdnl3Cnq396537HMIinZqDH8lBVVmYXcjE9D4M6RvdDkEaY8zZswRxNorXOKOVIqKd5JBwnl/FPtldTkF5HbMmWu3BGNPxBbSJqUsqynGeBNcz1umQjh/sd9EFqwvpHRnGdaMHBDBAY4xpG1aDaI29n8HfvgJR8TBnWauSQ2VtA29vKuHGzGQiw0MDGKQxxrQNSxD+KvgEXrwJovs5ySGudc1ES9YW09DsYebEDnLHtzHGnIElCH/kr4IXb4aYAU6fQ+znZv04rWOd02NTYhkxoHeAgjTGmLZlCeJMdr8PL97i1Bjm/AN6t77/YG3hIbaV1ljtwRjTqViCOJ2d78LLM6BPutMhHXN2dz4vWL2XqIhQpo4d2MYBGmNM4FiCOJUdK2D+LEgY6iSH6LN7ZkNNfSNvrt/P1DEDie5hg8aMMZ2HJQhftr0NC253ps2YvRR6JZz1W725fj9HGpuZafc+GGM6GUsQLW15CxbeAf1HOckhqs85vd2C7L0M7x/DuNS4NgrQGGPahyUIb3lvwCuznam673wdesaf09tt3lfFhqIqZk5MRcSeGmeM6VwsQRyz6VV45S5IHu88Q7rnuV/xL8wuJCIshBszWzcs1hhjOgJLEAAbXoFX74bUi+COVyHy3O9VqG9sZsnaYq4bnURcVEQbBGmMMe3LhtWUbYclc2HwJLh9IUT0apO3XbZxPzX1TcycYPc+GGM6J0sQfYfBzc/DsCkQ0XbPh16wupC0hCguHnJundzGGBMs1sQEMPqmNk0Ou8oOszq/ghkTBlnntDGm07IEEQALswsJCxFuHm+d08aYzssSRBtraPLwam4RV47oR7+YyGCHY4wxZ80SRBt7Z0sp5bUNNjGfMabTswTRxuav3svA2EguG3p2czcZY0xHYQmiDRVW1LFq50FuzUolNMQ6p40xnZsliDb0Sk4hALdNsIn5jDGdX0AThIhMEZFtIrJTRB7xsX+OiJSJyDr3dbfXvmav7UsDGWdbaPYoi3KKuGxoX5LjegY7HGOMOWcBu1FOREKBZ4CrgSIgW0SWqmpei0MXqur9Pt7iiKqOC1R8be2D7Qcoqa7n8Wkjgx2KMca0iUDWICYCO1V1t6o2AAuA6QH8vKCav7qQxOgIvpRxdk+dM8aYjiaQCSIZKPRaL3K3tXSziGwQkcUi4t14HykiOSLyqYh8xdcHiMhc95icsrKyNgy9dQ5U1/Pe1gPcPD6FiDDr1jHGdA3BPpu9CaSp6hhgBfCC177BqpoF3A78WkTOa1lYVZ9V1SxVzerbN3jDSl/JLaLZo8zIss5pY0zXEcgEUQx4nzFT3G3HqWq5qh51V58DxnvtK3b/7gbeBzIDGOtZ83iURTmFXJTehyF9o4MdjjHGtJlAJohsYKiIpItIBDATOGk0kogM8FqdBmxxt8eLSA93ORGYBLTs3O4QPt1dTkF5HbPszmljTBcTsFFMqtokIvcDy4FQYJ6qbhaRJ4EcVV0KPCAi04AmoAKY4xYfAfxJRDw4SewpH6OfOoT52YX0jgxjyuikYIdijDFtKqDPg1DVZcCyFtt+6LX8KPCoj3IfAxcEMra2UFnbwPJNJdx+0SAiw0ODHY4xxrSpYHdSd2qvrS2modnDDLtz2hjTBVmCOEuqysLsvYxNjWPEgHN/hrUxxnQ0liDO0pq9h9heephZVnswxnRRliDO0oLVe4mKCOWGsQODHYoxxgSEJYizUFPfyFsb9jNt7ECiewS0n98YY4LGEsRZWLp+H0cam61z2hjTpVmCOAsLVheSkRTDuNS4YIdijDEBYwmilTYVV7GxuIqZE1IRsafGGWO6LksQrbQwu5CIsBC+kulrYlpjjOk6LEG0wpGGZl5fV8z1o5OIi4oIdjjGGBNQNgSnFZZt3E9NfRMzJtjEfMYEWmNjI0VFRdTX1wc7lC4hMjKSlJQUwsPD/S5jCaIVFmTvJT2xFxcP6RPsUIzp8oqKioiJiSEtLc36+86RqlJeXk5RURHp6el+l7MmJj/tPHCY7PxKZljntDHtor6+noSEBPv31gZEhISEhFbXxixB+Glh9l7CQoSbL0wJdijGdBuWHNrO2fyWliD80NDk4dU1xVw1oj99Y3oEOxxjjGkXliD8sCKvlIraBmZMtDunjekuDh06xO9///tWl7v++us5dOhQACJqf5Yg/LAgey8DYyO5bGjfYIdijGknp0oQTU1Npy23bNky4uK6xiwLNorpDAor6vhox0EevHIooSHWHmpMMDzx5mby9lW36XuOHNibx6aOOuX+Rx55hF27djFu3DjCw8OJjIwkPj6erVu3sn37dr7yla9QWFhIfX09Dz74IHPnzgUgLS2NnJwcDh8+zHXXXcfkyZP5+OOPSU5O5o033qBnz55t+j0CyWoQZ7AopxARuM0m5jOmW3nqqac477zzWLduHT/72c9Ys2YNv/nNb9i+fTsA8+bNIzc3l5ycHJ5++mnKy8s/9x47duzgvvvuY/PmzcTFxfHqq6+299c4J1aDOI2mZg+v5BTxxWF9SY7rPFnfmK7mdFf67WXixIkn3UPw9NNPs2TJEgAKCwvZsWMHCQkJJ5VJT09n3LhxAIwfP578/Px2i7ctWA3iND7YXkZJdT0zrfZgTLfXq1ev48vvv/8+77zzDp988gnr168nMzPT5z0GPXqcGPUYGhp6xv6LjsYSxGnMX11IYnQEV47oH+xQjDHtLCYmhpqaGp/7qqqqiI+PJyoqiq1bt/Lpp5+2c3Ttw5qYTqG0up6V2w5w96XphIdaHjWmu0lISGDSpEmMHj2anj170r//iQvFKVOm8Mc//pERI0YwfPhwLr744iBGGjiWIE5hcW4RzR5lpk3MZ0y39fLLL/vc3qNHD/75z3/63HesnyExMZFNmzYd3/7d7363zeMLtIBeGovIFBHZJiI7ReQRH/vniEiZiKxzX3d77ZstIjvc1+xAxtmSx6MszC7k4iF9SE/sdeYCxhjTBQWsBiEiocAzwNVAEZAtIktVNa/FoQtV9f4WZfsAjwFZgAK5btnKQMXr7ZPd5eytqOM/rx7WHh9njDEdUiBrEBOBnaq6W1UbgAXAdD/LXgusUNUKNymsAKYEKM7Pmb96L7E9w5kyOqm9PtIYYzqcQCaIZKDQa73I3dbSzSKyQUQWi8ix8aR+lRWRuSKSIyI5ZWVlbRJ0RW0D/9pcyo2ZyUSGh7bJexpjTGcU7OE5bwJpqjoGp5bwQmsKq+qzqpqlqll9+7bNPEmvrSmiodnDTJuYzxjTzQUyQRQD3mfZFHfbcaparqpH3dXngPH+lg0EVadzelxqHBlJvQP9ccYY06EFMkFkA0NFJF1EIoCZwFLvA0RkgNfqNGCLu7wcuEZE4kUkHrjG3RZQa/ZWsuPAYbtz2hjTatHR0QDs27ePW265xecxl19+OTk5Oad9n1//+tfU1dUdXw/m9OEBSxCq2gTcj3Ni3wIsUtXNIvKkiExzD3tARDaLyHrgAWCOW7YC+BFOkskGnnS3BdT81YX0ighl6tiBgf4oY0wXNXDgQBYvXnzW5VsmiGBOHx7QG+VUdRmwrMW2H3otPwo8eoqy84B5gYzPW3V9I//YsJ/p4wbSq4fdP2hMh/LPR6BkY9u+Z9IFcN1Tp9z9yCOPkJqayn333QfA448/TlhYGCtXrqSyspLGxkZ+/OMfM336yYMz8/PzueGGG9i0aRNHjhzhrrvuYv369WRkZHDkyJHjx917771kZ2dz5MgRbrnlFp544gmefvpp9u3bxxVXXEFiYiIrV648Pn14YmIiv/zlL5k3zzkt3n333Tz00EPk5+cHbFrxYHdSdxhL1+3jSGMzMyfandPGGJgxYwaLFi06vr5o0SJmz57NkiVLWLNmDStXruQ73/kOqnrK9/jDH/5AVFQUW7Zs4YknniA3N/f4vp/85Cfk5OSwYcMGPvjgAzZs2MADDzzAwIEDWblyJStXrjzpvXJzc/nLX/7CZ599xqeffsqf//xn1q5dCwRuWnG7VHYtzC4kIymGsSmxwQ7FGNPSaa70AyUzM5MDBw6wb98+ysrKiI+PJykpiYcffpgPP/yQkJAQiouLKS0tJSnJ9z1TH374IQ888AAAY8aMYcyYMcf3LVq0iGeffZampib2799PXl7eSftbWrVqFTfeeOPxWWVvuukmPvroI6ZNmxawacUtQQCbiqvYWFzF41NHImJPjTPGOG699VYWL15MSUkJM2bM4KWXXqKsrIzc3FzCw8NJS0vzOc33mezZs4ef//znZGdnEx8fz5w5c87qfY5pOa24d1PWubAmJpxnTkeEhXBjZkqwQzHGdCAzZsxgwYIFLF68mFtvvZWqqir69etHeHg4K1eupKCg4LTlL7vssuMT/m3atIkNGzYAUF1dTa9evYiNjaW0tPSkif9ONc34pZdeyuuvv05dXR21tbUsWbKESy+9tA2/7ed1+xrEkYZm3li7j+tHJxEbFR7scIwxHcioUaOoqakhOTmZAQMG8NWvfpWpU6dywQUXkJWVRUZGxmnL33vvvdx1112MGDGCESNGMH68c6vX2LFjyczMJCMjg9TUVCZNmnS8zNy5c5kyZcrxvohjLrzwQubMmcPEiRMBp5M6MzMzoE+pk9N1sHQmWVlZeqbxxb6UVtfzo7fy+NolaUxM7xOAyIwxZ2PLli2MGDEi2GF0Kb5+UxHJVdUsX8d3+xpE/96R/O72C4MdhjHGdDjWB2GMMcYnSxDGmA6rqzSBdwRn81tagjDGdEiRkZGUl5dbkmgDqkp5eTmRkZGtKtft+yCMMR1TSkoKRUVFtNWzXrq7yMhIUlJaN5TfEoQxpkMKDw8nPT092GF0a9bEZIwxxidLEMYYY3yyBGGMMcanLnMntYiUAaefGOX0EoGDbRROZ2e/xcns9ziZ/R4ndIXfYrCq9vW1o8skiHMlIjmnut28u7Hf4mT2e5zMfo8TuvpvYU1MxhhjfLIEYYwxxidLECc8G+wAOhD7LU5mv8fJ7Pc4oUv/FtYHYYwxxierQRhjjPHJEoQxxhifun2CEJEpIrJNRHaKyCPBjieYRCRVRFaKSJ6IbBaRB4MdU7CJSKiIrBWRt4IdS7CJSJyILBaRrSKyRUQuCXZMwSQiD7v/TjaJyHwRad1UqZ1At04QIhIKPANcB4wEZonIyOBGFVRNwHdUdSRwMXBfN/89AB4EtgQ7iA7iN8DbqpoBjKUb/y4ikgw8AGSp6mggFJgZ3KjaXrdOEMBEYKeq7lbVBmABMD3IMQWNqu5X1TXucg3OCSA5uFEFj4ikAF8Gngt2LMEmIrHAZcDzAKraoKqHghtV0IUBPUUkDIgC9gU5njbX3RNEMlDotV5ENz4hehORNCAT+Cy4kQTVr4HvA55gB9IBpANlwF/cJrfnRKRXsIMKFlUtBn4O7AX2A1Wq+q/gRtX2unuCMD6ISDTwKvCQqlYHO55gEJEbgAOqmhvsWDqIMOBC4A+qmgnUAt22z05E4nFaG9KBgUAvEbkjuFG1ve6eIIqBVK/1FHdbtyUi4TjJ4SVVfS3Y8QTRJGCaiOTjND1+SUReDG5IQVUEFKnqsRrlYpyE0V1dBexR1TJVbQReA74Q5JjaXHdPENnAUBFJF5EInE6mpUGOKWhERHDamLeo6i+DHU8wqeqjqpqiqmk4/1+8p6pd7grRX6paAhSKyHB305VAXhBDCra9wMUiEuX+u7mSLthp360fOaqqTSJyP7AcZxTCPFXdHOSwgmkScCewUUTWudv+W1WXBTEm03F8G3jJvZjaDdwV5HiCRlU/E5HFwBqc0X9r6YLTbthUG8YYY3zq7k1MxhhjTsEShDHGGJ8sQRhjjPHJEoQxxhifLEEYY4zxyRKEMR2AiFxuM8aajsYShDHGGJ8sQRjTCiJyh4isFpF1IvIn93kRh0XkV+6zAd4Vkb7useNE5FMR2SAiS9z5exCR80XkHRFZLyJrROQ89+2jvZ638JJ7h64xQWMJwhg/icgIYAYwSVXHAc3AV4FeQI6qjgI+AB5zi/wN+C9VHQNs9Nr+EvCMqo7Fmb9nv7s9E3gI59kkQ3DubDcmaLr1VBvGtNKVwHgg27247wkcwJkOfKF7zIvAa+7zE+JU9QN3+wvAKyISAySr6hIAVa0HcN9vtaoWuevrgDRgVeC/ljG+WYIwxn8CvKCqj560UeT/a3Hc2c5fc9RruRn792mCzJqYjPHfu8AtItIPQET6iMhgnH9Ht7jH3A6sUtUqoFJELnW33wl84D6pr0hEvuK+Rw8RiWrXb2GMn+wKxRg/qWqeiPwA+JeIhACNwH04D8+Z6O47gNNPATAb+KObALxnP70T+JOIPOm+x63t+DWM8ZvN5mrMORKRw6oaHew4jGlr1sRkjDHGJ6tBGGOM8clqEMYYY3yyBGGMMcYnSxDGGGN8sgRhjDHGJ0sQxhhjfPp/tkaiPjXDi4kAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
}
},
{
"output_type": "display_data",
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhV1b3/8fc3MwmZSMKUAAFlCIQhCaOKgqhFxRGch6qt1tZW7U+9tb1t7W311ttaq63aFocqdaoFRxRpHRCpSJnnUZmSACGBkACBTOv3xzmEACEkkJOd5Hxez5OHk7332eebozmfrLX2Wtucc4iISPAK8boAERHxloJARCTIKQhERIKcgkBEJMgpCEREgpyCQEQkyCkIRBrIzF40s4cbeOwmMzvvVM8j0hwUBCIiQU5BICIS5BQE0qb4u2QeMLNlZrbPzJ43s05mNsPMSs3sIzNLrHX8pWa20syKzWyWmWXU2pdlZov8z/s7EHXUa00wsyX+535hZoNOsubbzWyDme0ys3fNrKt/u5nZ782swMxKzGy5mWX6911kZqv8teWZ2f0n9YaJoCCQtmkicD7QB7gEmAH8BEjB9//83QBm1gd4DbjXv+8D4D0zizCzCOBt4G9AB+Af/vPif24W8ALwHSAJ+AvwrplFNqZQMzsX+DVwNdAF2Ay87t99AXC2/+eI9x9T5N/3PPAd51wskAl80pjXFalNQSBt0R+dczucc3nA58A859xi59wB4C0gy3/cNcD7zrl/OecqgMeAdsAZwEggHHjCOVfhnJsKzK/1GncAf3HOzXPOVTnnXgIO+p/XGDcALzjnFjnnDgI/BkaZWTpQAcQC/QBzzq12zm3zP68C6G9mcc653c65RY18XZEaCgJpi3bUelxWx/ft/Y+74vsLHADnXDWwFUj178tzR67KuLnW4x7Aff5uoWIzKwa6+Z/XGEfXsBffX/2pzrlPgKeAp4ECM5tsZnH+QycCFwGbzewzMxvVyNcVqaEgkGCWj+8DHfD1yeP7MM8DtgGp/m2HdK/1eCvwiHMuodZXtHPutVOsIQZfV1MegHPuD865HKA/vi6iB/zb5zvnLgM64uvCeqORrytSQ0EgwewN4GIzG2dm4cB9+Lp3vgDmApXA3WYWbmZXAsNrPfdZ4E4zG+Ef1I0xs4vNLLaRNbwG3GpmQ/zjC/+Lrytrk5kN858/HNgHHACq/WMYN5hZvL9LqwSoPoX3QYKcgkCClnNuLXAj8EegEN/A8iXOuXLnXDlwJXALsAvfeMKbtZ67ALgdX9fNbmCD/9jG1vAR8DNgGr5WyGnAtf7dcfgCZze+7qMi4Lf+fTcBm8ysBLgT31iDyEkx3ZhGRCS4qUUgIhLkFAQiIkFOQSAiEuQUBCIiQS7M6wIaKzk52aWnp3tdhohIq7Jw4cJC51xKXftaXRCkp6ezYMECr8sQEWlVzGzz8fapa0hEJMgpCEREgpyCQEQkyLW6MQIRaVsqKirIzc3lwIEDXpfSJkRFRZGWlkZ4eHiDn6MgEBFP5ebmEhsbS3p6Okcu9iqN5ZyjqKiI3Nxcevbs2eDnqWtIRDx14MABkpKSFAJNwMxISkpqdOtKQSAinlMINJ2TeS+DJgg2FJTyP++tpLxSy7aLiNQWNEGwdVcZf/33Jj5dW+B1KSLSghQXF/PMM880+nkXXXQRxcXFAaio+QVNEIzunUxKbCRTF+Z6XYqItCDHC4LKysp6n/fBBx+QkJAQqLKaVdAEQVhoCFdkpfLpmgKK9h70uhwRaSEefPBBvvrqK4YMGcKwYcMYPXo0l156Kf379wfg8ssvJycnhwEDBjB58uSa56Wnp1NYWMimTZvIyMjg9ttvZ8CAAVxwwQWUlZV59eOclKC6fHRidhqTZ3/NO0vyue2shl9aJSLN43/eW8mq/JImPWf/rnE8dMmA4+5/9NFHWbFiBUuWLGHWrFlcfPHFrFixoubyyxdeeIEOHTpQVlbGsGHDmDhxIklJSUecY/369bz22ms8++yzXH311UybNo0bb7yxSX+OQAqaFgFA386xZKbGMW2RuodEpG7Dhw8/4hr8P/zhDwwePJiRI0eydetW1q9ff8xzevbsyZAhQwDIyclh06ZNzVVukwhYi8DMugFTgE6AAyY755486hgDngQuAvYDtzjnFgWqJoBJ2Wn84r1VrN5WQkaXuEC+lIg0Un1/uTeXmJiYmsezZs3io48+Yu7cuURHRzNmzJg6r9GPjIyseRwaGtrquoYC2SKoBO5zzvUHRgJ3mVn/o465EOjt/7oD+FMA6wHg0iGphIca0zRoLCJAbGwspaWlde7bs2cPiYmJREdHs2bNGr788stmrq55BCwInHPbDv1175wrBVYDqUcddhkwxfl8CSSYWZdA1QTQISaCc/t15O0l+VRUaU6BSLBLSkrizDPPJDMzkwceeOCIfePHj6eyspKMjAwefPBBRo4c6VGVgdUsg8Vmlg5kAfOO2pUKbK31fa5/27ajnn8HvhYD3bt3P+V6JmanMXPlDmav28m4jE6nfD4Rad1effXVOrdHRkYyY8aMOvcdGgdITk5mxYoVNdvvv//+Jq8v0AI+WGxm7YFpwL3OuZO6HMA5N9k5N9Q5NzQlpc47rTXKmL4d6RAToUFjERECHARmFo4vBF5xzr1ZxyF5QLda36f5twVURFgIlw3pykerCijeXx7olxMRadECFgT+K4KeB1Y75x4/zmHvAjebz0hgj3Nu23GObVITs9Mor6rmvaX5zfFyIiItViBbBGcCNwHnmtkS/9dFZnanmd3pP+YD4GtgA/As8L0A1nOEAV3j6Nc5lqmLAt4AERFp0QI2WOycmwPUux6qc84BdwWqhvqYGZNy0nj4/dVsKCjl9I6xXpQhIuK5oJpZfLTLhqQSGmJMXahWgYgEr6AOgpTYSMb0SeGtxblUVTuvyxGRVqB9+/YA5OfnM2nSpDqPGTNmDAsWLKj3PE888QT79++v+d7LZa2DOggAJuaksaPkIHM2FHpdioi0Il27dmXq1Kkn/fyjg8DLZa2DPgjGZXQkvl24lpwQCVIPPvggTz/9dM33v/jFL3j44YcZN24c2dnZDBw4kHfeeeeY523atInMzEwAysrKuPbaa8nIyOCKK644Yq2h7373uwwdOpQBAwbw0EMPAb6F7PLz8xk7dixjx44FDi9rDfD444+TmZlJZmYmTzzxRM3rBWq566BahroukWGhXDq4K28s2ErJgQriosK9LkkkeM14ELYvb9pzdh4IFz563N3XXHMN9957L3fd5btu5Y033mDmzJncfffdxMXFUVhYyMiRI7n00kuPez/gP/3pT0RHR7N69WqWLVtGdnZ2zb5HHnmEDh06UFVVxbhx41i2bBl33303jz/+OJ9++inJyclHnGvhwoX89a9/Zd68eTjnGDFiBOeccw6JiYkBW+466FsE4OseOlhZzfvLmmUKg4i0IFlZWRQUFJCfn8/SpUtJTEykc+fO/OQnP2HQoEGcd9555OXlsWPHjuOeY/bs2TUfyIMGDWLQoEE1+9544w2ys7PJyspi5cqVrFq1qt565syZwxVXXEFMTAzt27fnyiuv5PPPPwcCt9x10LcIAAanxXN6x/ZMW5jLdcNPfS0jETlJ9fzlHkhXXXUVU6dOZfv27VxzzTW88sor7Ny5k4ULFxIeHk56enqdy0+fyMaNG3nssceYP38+iYmJ3HLLLSd1nkMCtdy1WgT45hRMzE5jwebdbCzc53U5ItLMrrnmGl5//XWmTp3KVVddxZ49e+jYsSPh4eF8+umnbN68ud7nn3322TUL161YsYJly5YBUFJSQkxMDPHx8ezYseOIBeyOt/z16NGjefvtt9m/fz/79u3jrbfeYvTo0U340x5LQeB3RVYqIQZvaiE6kaAzYMAASktLSU1NpUuXLtxwww0sWLCAgQMHMmXKFPr161fv87/73e+yd+9eMjIy+PnPf05OTg4AgwcPJisri379+nH99ddz5pln1jznjjvuYPz48TWDxYdkZ2dzyy23MHz4cEaMGMG3v/1tsrKymv6HrsV8k3tbj6FDh7oTXZ97sm5+4T98VbCXz/9rLCEh9U6KFpEmsnr1ajIyMrwuo02p6z01s4XOuaF1Ha8WQS0Ts1PJKy7jy6+LvC5FRKTZKAhq+caAzsRGhjFV3UMiEkQUBLVEhYcyYXAXZizfzt6DlV6XIxI0WlsXdUt2Mu+lguAok3LSKKuoYsZyzSkQaQ5RUVEUFRUpDJqAc46ioiKioqIa9TzNIzhKdvdEeibHMG1RLlcN7XbiJ4jIKUlLSyM3N5edO3d6XUqbEBUVRVpaWqOeoyA4iplxZVYqv/vXOrbu2k+3DtFelyTSpoWHh9OzZ0+vywhq6hqqw5U5aZjBm7p7mYgEAQVBHVIT2jGqVxLTFuWq31JE2jwFwXFMykljy679zN+02+tSREQCSkFwHOMzOxMTEar7FIhIm6cgOI7oiDAuHNiF95dvo6y8yutyREQCJmBBYGYvmFmBma04zv54M3vPzJaa2UozuzVQtZysSTlp7D1YycyV270uRUQkYALZIngRGF/P/ruAVc65wcAY4HdmFhHAehpteHoH0hLbMU1LTohIGxawIHDOzQZ21XcIEGu+e7+19x/botZ1CAnx3adgzoZC8oub5gYQIiItjZdjBE8BGUA+sBy4xzlXXdeBZnaHmS0wswXNPftwYnYazsFbizWnQETaJi+D4BvAEqArMAR4yszi6jrQOTfZOTfUOTc0JSWlOWuke1I0w9M7MG2h5hSISNvkZRDcCrzpfDYAG4H6bwPkkUk5aXxduI/FW4u9LkVEpMl5GQRbgHEAZtYJ6At87WE9x3XhwM5EhYdoToGItEmBvHz0NWAu0NfMcs3sW2Z2p5nd6T/kV8AZZrYc+Bj4kXOuMFD1nIrYqHAuzOzCe0vzOVChOQUi0rYEbPVR59x1J9ifD1wQqNdvahOz03hrcR4frd7BhEFdvS5HRKTJaGZxA406LYku8VFMVfeQiLQxCoIGCg0xrsxOZfa6nRSUHPC6HBGRJqMgaIQrs9OodvD2Es0pEJG2Q0HQCKeltCe7ewJTNadARNoQBUEjTcxJY92OvazIK/G6FBGRJqEgaKQJg7oSERaihehEpM1QEDRSfLtwLujfiXeW5FFeWefSSCIirYqC4CRMzElj9/4KPllT4HUpIiKnTEFwEkafnkxKbKTmFIhIm6AgOAlhoSFcmZXKrLUFFO096HU5IiKnREFwkibmpFFZ7XhnSb7XpYiInBIFwUnq0ymWQWnx6h4SkVZPQXAKJmansWpbCavyNadARFovBcEpuHRwV8JDTXMKRKRVUxCcgsSYCMb1880pqKjSnAIRaZ2CJwiqq2H9R01+2ok5aRTuLWf2up1Nfm4RkeYQPEGweAq8MhHmPNGkpx3TN4WkmAgNGotIqxU8QTDkRsicCB89BJ8/3mSnDQ8N4bIhqXy8uoDi/eVNdl4RkeYSPEEQGgZXTIbMSfDx/8Dnv2uyU0/MSaW8qpr3lmpOgYi0PsETBOAPg7/AwKvh41/C7N82yWkHdI0no0ucuodEpFUKriAAfxj8GQZdC588DJ/9pklOOzE7laW5e1i/o7RJzici0lyCLwgAQkLh8mdg8HXw6SMw69FTPuXlWamEhRhTNadARFqZgAWBmb1gZgVmtqKeY8aY2RIzW2lmnwWqljqFhMJlT8OQG2DWr+HT/4VTuP1kcvtIxvRN4e3FeVRV6zaWItJ6BLJF8CIw/ng7zSwBeAa41Dk3ALgqgLXULSQULn0Ksm6Ez/7vlMNgYnYaO0oOMmdDYRMWKSISWAELAufcbGBXPYdcD7zpnNviP96bu7yEhMAlf4Tsm2H2b3zjBicZBudmdCQhOlyDxiLSqoR5+Np9gHAzmwXEAk8656bUdaCZ3QHcAdC9e/emryQkBCY8CRh8/hi4ahj3czBr1Gkiw0K5dHBX/j5/K3vKKohvF970tYqINDEvB4vDgBzgYuAbwM/MrE9dBzrnJjvnhjrnhqakpASmmpAQmPAE5NwKcx6Hj35xUi2DSTlpHKys5v1l25q+RhGRAPAyCHKBmc65fc65QmA2MNjDenxhcPHjMPRb8O8n4F8/b3QYDEyNp3fH9lqRVERaDS+D4B3gLDMLM7NoYASw2sN6fEJC4OLfwbDb4Ys/wD9/2qgwMDMm5qSxcPNuNhbuC2ChIiJNI5CXj74GzAX6mlmumX3LzO40szsBnHOrgQ+BZcB/gOecc8e91LRZmcFFv4Xh34G5T8HM/25UGFyRlUqIwTQNGotIKxCwwWLn3HUNOOa3QNOs89DUzODC/wMLgS+f9g0gj/91gwaQO8VFMbp3Cm8uyuX/nd+HkJDGDTqLiDSn4JxZ3FBmvg//kd+DeX+CGT9qcMtgUk4a+XsOMPfrogAXKSJyary8fLR1MINv/K+vZTD3KV/L4KLfnrBlcH7/TsRGhTFtYS5nnp7cTMWKiDSeWgQNYQYXPAxn/ADmPwvv3+e741k9osJDmTCoKzNWbGfvwcpmKlREpPEUBA1lBuf/Cs68BxY8Dx+cOAwm5aRRVlHFB8s1p0BEWi4FQWOYwXn/A2f9EBa8AO//sN4wyO6eQM/kGF09JCItmoKgscxg3EMw+j5Y+CJMv+e4YWBmTMpJY97GXWzdtb956xQRaSAFwckwg3N/Bmc/AIumwHs/OG4YXJGVihmaaSwiLZaC4GSZwdj/hnN+BItfhne/D9VVxxzWNaEdZ5yWxLRFuVTrPgUi0gIpCE6FGYz9CYz5MSx5Bd6pOwwm5aSxdVcZ8zfVtyq3iIg3FARNYcyDMOYnsPRVePt7x4TBNwZ0JiYiVN1DItIiKQiaypgfwdifwrLX4a07jwiD6IgwLh7UhQ+Wb2d/ueYUiEjLoiBoSuc84BtEXv4GvPUdqDr8oT8xO429ByuZuXK7hwWKiBxLQdDUzr7fd3np8n/AW3fUhMGw9A5069COaQvzPC5QRORICoJAGP3/4LxfwIpp8Oa3oaqSkBBjYnYa//6qkPziMq8rFBGpoSAIlLN+COf/Ela+BdO+BVUVTMxOwzl4a7FaBSLScigIAunMe3yL1a16G6beRrf4cEb07MDf52/VQnQi0mIoCALtjB/4lrFe/S784xbuOrs7ecVl3PT8PPaUVXhdnYiIgqBZjLoLxj8Ka6Zz9tL/4plrM1mRt4frn/2SXfvKva5ORIJcg4LAzO4xszjzed7MFpnZBYEurk0Z+V248DewZjrfWHovr14ax4aCvVzzl7kUlBzwujoRCWINbRHc5pwrAS4AEoGbgEcDVlVbNeI7MOH3sGUuw2ZczLzuz9C9eB5X//kL8nQlkYh4pKFBcOi+jBcBf3POray1TRpj6G3ww5Vw7s9I2LOG50Me4c/77+W5p3/N5oJir6sTkSDU0CBYaGb/xBcEM80sFqj39lxm9oKZFZjZihMcN8zMKs1sUgNraf2iO/gmnv1wBVz2NOkJETxU8SRRz2RROPM3cGCP1xWKSBBpaBB8C3gQGOac2w+EA7ee4DkvAuPrO8DMQoH/A/7ZwDralrBIyLqRqHvmk3vRFDZZKslzH6Hqdxnw4U+geIvXFYpIEGhoEIwC1jrnis3sRuCnQL1/tjrnZgMnWnf5B8A0oKCBdbRNZqQNv4yU733IN8MfY0ZFFm7en+HJITD1W5C/2OsKRaQNa2gQ/AnYb2aDgfuAr4App/LCZpYKXOE/twC9Utrz8Hdv4DfR93N+9R/Z1v82WDcTJo+BFyfA2g/rvUeyiMjJaGgQVDrnHHAZ8JRz7mkg9hRf+wngR865E36ymdkdZrbAzBbs3LnzFF+2ZevWIZo3vjMKF5fK2GXj+OKy2b7Zybs2wmvXwDMjYOFLUKFLTkWkaTQ0CErN7Mf4Lht938xC8I0TnIqhwOtmtgmYBDxjZpfXdaBzbrJzbqhzbmhKSsopvmzL1zk+ir9/ZxTpSTHc8upaPkq4Gu5ZAlc+B2FR8N7d8EQmfPYb2Ffkdbki0so1NAiuAQ7im0+wHUgDfnsqL+yc6+mcS3fOpQNTge85594+lXO2JcntI3n9jpFkdInlzpcX8v7KQhh0FXxnNtz8LnTNgk8fgd8PgPfvg6KvvC5ZRFqpBgWB/8P/FSDezCYAB5xz9Y4RmNlrwFygr5nlmtm3zOxOM7vzlKsOEgnREbz87RFkdU/gB68tYtrCXN99knudAzf8A773JQycCIumwB9z4PUbYMs8r8sWkVbGfF3/JzjI7Gp8LYBZ+CaSjQYecM5NDWh1dRg6dKhbsGBBc7+sp/aXV3L7lAX8e0MRj1yRyQ0jehx5QOkO+M9kmP8cHCiGtGG+xe76TYCQUG+KFpEWxcwWOueG1rmvgUGwFDjfOVfg/z4F+Mg5N7hJK22AYAwCgAMVVXzvlUV8sqaAn16cwbdH9zr2oPJ9sORVmPsU7N4Eiekw8i7IugEiYpq7ZBFpQeoLgoaOEYQcCgG/okY8V5pAVHgof74xh4sGdubh91fz1Cfrjz0oIgaG3w4/WARXT4GYFJjxADzeHz7+JZTqfskicqywBh73oZnNBF7zf38N8EFgSpLjiQgL4Q/XZhEVtozH/rmO/eVVPPCNvpgdtexTSCj0v8z3tWUefPEH+Pxx+OKPMOhqGPV96JjhzQ8hIi1Og4LAOfeAmU0EzvRvmuyceytwZcnxhIWG8NhVg4kMD+WZWV+xv7yKhy7pf2wYHNJ9BHR/xXdV0ZfPwOJXYPHLcPr5kH2zbzwhrkvz/hAi0qI0aIygJQnWMYKjOef41fTVvPDvjVw3vBsPXz6Q0JAGLAi7rwgWPO8bXN7nn5wX2xVSs/1fOb5LU6PiA/sDiEizqm+MoN4WgZmVAnUlhQHOORfXBPXJSTAzfjYhg+iIUJ76dANl5VU8dtVgwkJPMHQTkwTn/JfvfsrblkLeIshb6PtaM/3wcUm9faGQmuMLiE6ZEB4V2B9KRDxRbxA45051GQkJIDPj/m/0pV1EKL+duZaDldU8eW0WEWENGMcPi4Ruw31fh5Tt9i1wl7fQFxBffwrLXvftCwmHzpnQNftwOCT30eWpIm2AuobaiOfnbORX01cxtm8Kf7oxh6jwJviAdg5K8n3BkH+o5bAYykt9+yPa+7qRumYdbj3Ep/kmvYlIi3LK8whaEgXB8b06bwv//fZyRvVK4tmbhxIT2dCLwhqhuhqKNhzuTspfBNuXQ1W5b39MyuFQ6Oofd4ju0PR1iEijKAiCyFuLc7nvjaVkdU/kr7cOIy7qVNcGbIDKg7BjhX+8wd9yKFxHzfBSYs9aA9HZ0GUwREQHvi4RqaEgCDIfLN/G3a8tJqNLHFNuG05iTETzF3GgBLYtOTzekLcISnJ9+ywUOvaHVH+XUmI6RCX4rlRqlwCR8RCi+YoiTUlBEIQ+Xr2D776yiJ5JMbz87RGkxEZ6XZJvTaT8Wlcp5S3yrY10DIOoOF8wRCX4wqF2UNQ8Tqy1v9axoc3QChJpZRQEQWrO+kJun7KALvFRvHL7CLrEt/O6pCM5B7s3Qsk2XyCUFcOBPbUe+78/9PjQ/sqy+s8bHnNkaBwdFMcLmHaJEN7C3iORJqIgCGLzN+3i1r/OJzEmnFe/PZJuHdpA33zlwbqD4pjQqGP/wZL6zx0W5Q+JRP/XUY+P2Fdrv7qzpIVTEAS5pVuLufmF/9AuPJRXbh/BaSntvS7JO1WVvjAo231s66Os2L/d/2/ZUf9W7KvnxHa4u6rOEEk8fsCEtYBuO2l6zvmupqsqh6oK37+VBw8/rr29qo7tlQePOqYCug2DXmNOqhwFgbB6Wwk3PjcPM3j52yPo11mTwhutsrxWSOw+Kix2HxUiu48MlvpuzR0efTgkouLBTrVl0QS/0xZybBfa8f6NiocwDy5IOFWH/ig4oltyz+HW46GvsmKoKKv1IV1exwd8HduqK5q+5jPuhgt+dVJPVRAIABsK9nLDc19ysLKav902goFpWk+oWVRX1/rAOU6I1P4waorfyVOd1FddeWS3WsX++o+vCbP6QuM4wXKyS5dUV/smN9b+wK79AV7XB3rtfeV76z+/hR6uOyIGQiP8X+G1HkfUsT3c18o75thDjyPr3h5W17mO+goJPen/tgoCqbG5aB/XPzuPkrIKnr9lGMN7arKXNEBNa+iogftjttVxzKGZ6MdTMy5TR1iEhh/5gV77w/xgSf0tLfCN3bSLP3zeI/6Nr/WaRx/j//BvQ7PkFQRyhPziMm54bh6bivZxzdBu3HdB35Zxeam0TVWVR4XG7voH9Wv+3ePrXmnoB/cR+xIgMlZrYdWiIJBj7Cmr4I8fr+fFLzYRFR7K9889nVvPTCcyTL84Im1RU9yqUtqY+Hbh/HRCf/75w7MZ2asDj85Yw/mPz+bDFdtobX8ciMipURAEuV4p7Xnum8OYcttwosJDuPPlRVz37JeszN/jdWki0kwCFgRm9oKZFZjZiuPsv8HMlpnZcjP7wswGB6oWObGz+6Twwd2j+dXlmazdXsqEP87hx28uY2fpQa9LE5EAC2SL4EVgfD37NwLnOOcGAr8CJgewFmmAsNAQbhrZg1n3j+W2M3vyjwW5jH1sFn/+7CsOVlZ5XZ6IBEjAgsA5NxvYVc/+L5xzu/3ffgmkBaoWaZz46HB+5h8/GNGz9vjBdo0fiLRBLWWM4FvAjOPtNLM7zGyBmS3YuXNnM5YV3HqltOf5W2qPHyzk+mfnsSr/BOv1iEirEtDLR80sHZjunMus55ixwDPAWc65ohOdU5ePeqOyqprX/rOFx/+1juKyCq4d5pt/kNxe8w9EWoMWe/momQ0CngMua0gIiHfCQkO4aVQ6s+4fy61n+MYPxvx2Fn/R+IFIq+dZEJhZd+BN4Cbn3Dqv6pDGiY8O5+eX9GfmD89meM8O/HrGGi74/WxmrtT4gUhrFbCuITN7DRgDJAM7gIeAcADn3J/N7DlgIrDZ/5TK4zVbalPXUMvy2bqdPDx9FesL9jKqVxI/v6Q/GV20sqlIS6MlJiSgKquqedU/flBSVsE1w7pz3wV9NH4g0oK02DECaRvCQkO4eVQ6n90/lrv/bTAAABERSURBVG+ekc4/Fmxl7G9nMXm2xg9EWgMFgTSZ+OhwHrpkAB/eezbDenbgfz/wjR/8U+MHIi2agkCa3Okd2/PCLcN46bbhhIeGcMffFnLDc/NYvU3zD0RaIgWBBMw5fVL48J7R/PKyAazaVsLFf/icH7+5nMK9Wr9IpCVREEhAHRo/mHX/mCPGD56d/TXllSe4u5SINAsFgTSLhOiImvGDoemJPPLBai74/WcaPxBpARQE0qxO79iev946nBdvHUaYf/zg8me+4K3FubrCSMQjmkcgnqmoqubv87fy/JyNbCzcR3L7CK4b3p3rR3SnS3w7r8sTaVM0oUxatOpqx5wNhbz0xSY+WVtAiBnjB3Tm5lE9GN6zA2bmdYkirV59QRDW3MWIHC0kxDi7Twpn90lhS9F+Xp63mb/P38r7y7fRr3MsN49K5/KsrkRH6H9XkUBQi0BapLLyKt5ZksdLczezelsJcVFhXD20GzeN6kGPpBivyxNpddQ1JK2Wc44Fm3fz0heb+HDFdqqcY2zfjtw8qgdn904hJETdRiINoa4habXMjGHpHRiW3oHtew7w6n+28Oq8Ldzy1/mkJ0Vz06h0JuWkEd8u3OtSRVottQik1SmvrGbGim289MUmFm0pJjoilCuyUrl5VDp9O8d6XZ5Ii6SuIWmzlufuYcrcTbyzNJ/yympG9urALWekc15GJ8JCNU1G5BAFgbR5u/aV8/f5W3n5y83kFZfRJT6KG0f24Jph3XRfBBEUBBJEqqodH6/ewUtzN/HvDUVEhIYwYXAXvjkqncHdErwuT8QzGiyWoBEaYlwwoDMXDOjMhoJSpszdzLSFuby5KI/B3RK45YweXDSwC5FhoV6XKtJiqEUgbV7pgQreXJTHS3M38fXOfSTF+JayuGGklrKQ4KGuIRF8S1n8+6tCXvpiMx+v2UGIGRf078Q3z0hnhJaykDZOXUMi+JayGN07hdG9U9i6az8vf7mZ1+dvZcaK7fTtFMuknDQuHtSFrglqJUhwUYtAglpZeRXvLc3n5XmbWZa7B4ChPRKZMKgLFw3sQse4KI8rFGkannQNmdkLwASgwDmXWcd+A54ELgL2A7c45xad6LwKAgmUjYX7eH9ZPtOXbWPN9lLMYETPDkwY1JULMzuTpMtQpRXzKgjOBvYCU44TBBcBP8AXBCOAJ51zI050XgWBNIcNBaW8t3Qb05fl89XOfYSGGGeclsSEQV34xoDOJERHeF2iSKN4NlhsZunA9OMEwV+AWc651/zfrwXGOOe21XdOBYE0J+cca7aXMt3fUthctJ+wEGN072QmDOrK+QM6EReldY6k5Wupg8WpwNZa3+f6tx0TBGZ2B3AHQPfu3ZulOBHwLXqX0SWOjC5x3H9BX1bkldSEwn3/WErEmyGc0zeFCYO6cF5GJ2Iidf2FtD6t4v9a59xkYDL4WgQelyNByswYmBbPwLR4HrywH4u3FjN96TbeX57Pv1btICo8hHP7dWTCoK6M7duRdhGatCatg5dBkAd0q/V9mn+bSItnZmR3TyS7eyI/vTiD+Zt2MX3ZNmas2MYHy7cTHRHKeRmdmDCoC+f0TdFMZmnRvAyCd4Hvm9nr+AaL95xofECkJQoJMUb0SmJEryQeuqQ/8zbuYvqyfGas2M67S/OJjQzj/AGduGRQV848PZmIMK2KKi1LIK8aeg0YAyQDO4CHgHAA59yf/ZePPgWMx3f56K3OuROOAmuwWFqLiqpq/r2hkOnLtjFz5XZKD1QS3y6c8QM6M2FwF0b1StJS2dJstMSEiMcOVlbx+bpCpi/zjSfsK68iKSaC8ZmdmTCoK8N7diBUt92UAFIQiLQgByqqmLW2gPeWbePj1Ts4UFFNx9hILhrYhQmDupDVPVGhIE1OQSDSQu0vr+Tj1QVMX5bPp2t3Ul5ZTXy7cM44Lcm/LlIy3TpEe12mtAEtdR6BSNCLjgjjksFduWRwV0oPVPDp2p18vm4nczYUMmPFdgDSk6I5q3cyo3unMOq0JE1gkyanFoFIC+Sc46ude/l8fSGfry/ky6+L2F9eRWiIMaRbAmednszo3skM6ZagAWdpEHUNibRy5ZXVLN6y2xcMGwpZlluMcxAbGcbI05I4u3cyZ/VOIT0pWvdVkDopCETamOL95XzxVRGfr9/J5+sLyd1dBkBaYjtG+7uRzjgtSYvjSQ0FgUgb5pxjc9H+mlCY+1URpQcrMYNBaQmMPj2Zs3onk909UZPZgpiCQCSIVFZVszS3mNnrCpmzoZAlW4upqnZER4QysleSv8WQzGkp7dWNFEQUBCJBrORABXO/KmLOel8wbCzcB0CX+CjfoHOfFM48LUk33mnjFAQiUmPrrv3M2VDI5+t38u8NRewpqwBgQNe4mrkLOT0SiQrXQnltiYJAROpUVe1YkbenZnxh0ZbdVFQ5IsJCGJwWT06PDuT0SCSnRyIdYjTw3JopCESkQfYdrGTexiK+2FDEwi27WZG3h4oq32dEr+QYcnokMjTdFwy9ktsToqUwWg0FgYiclAMVVSzP28OCTbtZuHk3CzfvYvd+X1dSQnQ42d0Ta1oMg9MSdDOeFkxLTIjISYkKD2VYegeGpXcAfJeqbizcx4LNu1m0eTcLNu/mkzUFAISFGAO6xtV0Jw1NT6RTXJSX5UsDqUUgIqekeH85i7bsrmk1LM0t5kBFNQCpCe0Ymp7I0B6JZPdIpF/nOK2s6hG1CEQkYBKiIzi3XyfO7dcJ8N2QZ1V+SU2r4cuvi3hnST4AMRGhZNXqTsrqnkCsFtHznFoEIhJQzjnyisv8Ywy+lsOa7SVUOzCDvp1iawagh/boQFpiO010CwANFotIi1J6oIKlW/ewYPMuFm7ezeItxew9WAlASmwkQ2taDIn07xKnQegmoK4hEWlRYqPCOau3bw0k8M1nWLejtNYg9K6a+zGEGPTuGMuA1DgGpsYzMDWejC5xxETq46upqEUgIi1SQckBlmwtZkV+CSvy9rA8bw87Sw8Cvi6l01LaMzA1ngFdfQHRv2ucxhvqoa4hEWkTdpQcqAmFFXl7WJFXwvaSA4AvHHomxZDpbzUMSI0jMzVed3Tz86xryMzGA08CocBzzrlHj9rfHXgJSPAf86Bz7oNA1iQirVenuCg6xUUxLqNTzbadpQf9obDHP/ltF+8uza/Zn54UzQB/OBxqQeg+DUcKWIvAzEKBdcD5QC4wH7jOObeq1jGTgcXOuT+ZWX/gA+dcen3nVYtARE6kaO/Bmi6lQwFx6OY9AN06tCOza3xN6yEzNb7Nr6XkVYtgOLDBOfe1v4jXgcuAVbWOcUCc/3E8kI+IyClKah/JOX1SOKdPSs223fvKWZHv6046FA6HBqTBN/ktMzXOFxBpvoBIDpKluQMZBKnA1lrf5wIjjjrmF8A/zewHQAxwXl0nMrM7gDsAunfv3uSFikjblxgT4V9m+3A47Nlfwcp8/5iDvwUxc+WOmv2d46LI9A9E9+nUnr6dYklPjiE8tG3d6c3r66+uA150zv3OzEYBfzOzTOdcde2DnHOTgcng6xryoE4RaYPio8M54/Rkzjg9uWZbyYEKVtW6Uml53h4+WbODav8nT3io0Su5Pb39wdC7Uyx9O8fSvUN0q10+I5BBkAd0q/V9mn9bbd8CxgM45+aaWRSQDBQEsC4RkeOKiwpnZK8kRvZKqtl2oKKKDQV7WV9Qytrte1m/o5QlW4uZvmxbzTGRYSGc3rE9fTrF+r98j1MT2rX45boDGQTzgd5m1hNfAFwLXH/UMVuAccCLZpYBRAE7A1iTiEijRYWHkukfVK5t38FK1hfsZd2OUtZtL2VdwV7mflXEW4sP/80bHRFK706x9OnYnr6d/S2ITrF0iotsMUtpBCwInHOVZvZ9YCa+S0NfcM6tNLNfAgucc+8C9wHPmtkP8Q0c3+Ja28QGEQlaMZFhDOmWwJBuCUds31NWwfodpazb4Q+JHaV8uraAfyzMrTkmNirscNeSv/XQp3OsJwPUmlAmItJMivYeZN2OQ11MpazfsZe1O0pr7hsN0CEmoqZbqXY306nOfdBaQyIiLUBS+0hGtY9k1GmHxx+cc+wsPcjaQy2I7aWsKyhl2sJc9pVX1RzXMTaS20f34vazezV5XQoCEREPmRkd46LoGBd1xKWtzjny9xzwBcOOUtbuKKVjXGC6jRQEIiItkJmRmtCO1IR2jO3XMaCv1bZmRYiISKMpCEREgpyCQEQkyCkIRESCnIJARCTIKQhERIKcgkBEJMgpCEREglyrW2vIzHYCm0/y6clAYROW09rp/TiS3o/D9F4cqS28Hz2ccyl17Wh1QXAqzGzB8RZdCkZ6P46k9+MwvRdHauvvh7qGRESCnIJARCTIBVsQTPa6gBZG78eR9H4cpvfiSG36/QiqMQIRETlWsLUIRETkKAoCEZEgFzRBYGbjzWytmW0wswe9rsdLZtbNzD41s1VmttLM7vG6Jq+ZWaiZLTaz6V7X4jUzSzCzqWa2xsxWm9kor2vyipn90P87ssLMXjOzKK9rCoSgCAIzCwWeBi4E+gPXmVl/b6vyVCVwn3OuPzASuCvI3w+Ae4DVXhfRQjwJfOic6wcMJkjfFzNLBe4GhjrnMoFQ4FpvqwqMoAgCYDiwwTn3tXOuHHgduMzjmjzjnNvmnFvkf1yK7xc91duqvGNmacDFwHNe1+I1M4sHzgaeB3DOlTvnir2tylNhQDszCwOigXyP6wmIYAmCVGBrre9zCeIPvtrMLB3IAuZ5W4mnngD+C6j2upAWoCewE/irv6vsOTOL8booLzjn8oDHgC3ANmCPc+6f3lYVGMESBFIHM2sPTAPudc6VeF2PF8xsAlDgnFvodS0tRBiQDfzJOZcF7AOCckzNzBLx9Rz0BLoCMWZ2o7dVBUawBEEe0K3W92n+bUHLzMLxhcArzrk3va7HQ2cCl5rZJnxdhuea2cveluSpXCDXOXeohTgVXzAEo/OAjc65nc65CuBN4AyPawqIYAmC+UBvM+tpZhH4Bnze9bgmz5iZ4esDXu2ce9zrerzknPuxcy7NOZeO7/+LT5xzbfKvvoZwzm0HtppZX/+mccAqD0vy0hZgpJlF+39nxtFGB87DvC6gOTjnKs3s+8BMfCP/LzjnVnpclpfOBG4ClpvZEv+2nzjnPvCwJmk5fgC84v+j6WvgVo/r8YRzbp6ZTQUW4bvSbjFtdKkJLTEhIhLkgqVrSEREjkNBICIS5BQEIiJBTkEgIhLkFAQiIkFOQSDSjMxsjFY4lZZGQSAiEuQUBCJ1MLMbzew/ZrbEzP7iv1/BXjP7vX99+o/NLMV/7BAz+9LMlpnZW/41ajCz083sIzNbamaLzOw0/+nb11rv/xX/rFURzygIRI5iZhnANcCZzrkhQBVwAxADLHDODQA+Ax7yP2UK8CPn3CBgea3trwBPO+cG41ujZpt/exZwL757Y/TCN9NbxDNBscSESCONA3KA+f4/1tsBBfiWqf67/5iXgTf96/cnOOc+829/CfiHmcUCqc65twCccwcA/Of7j3Mu1//9EiAdmBP4H0ukbgoCkWMZ8JJz7sdHbDT72VHHnez6LAdrPa5Cv4fiMXUNiRzrY2CSmXUEMLMOZtYD3+/LJP8x1wNznHN7gN1mNtq//SbgM/+d33LN7HL/OSLNLLpZfwqRBtJfIiJHcc6tMrOfAv80sxCgArgL301ahvv3FeAbRwD4JvBn/wd97dU6bwL+Yma/9J/jqmb8MUQaTKuPijSQme11zrX3ug6RpqauIRGRIKcWgYhIkFOLQEQkyCkIRESCnIJARCTIKQhERIKcgkBEJMj9f5/f8J8YiaFeAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
}
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Uw6YR76p_AF0"
},
"source": [
"## Saving models"
]
},
{
"cell_type": "code",
"source": [
"name = \"cnn_conv1D_egc_\"+embedding_name+\"_s\"+str(maxOfInstancePerClass)"
],
"metadata": {
"id": "irB4mATeeUcw"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "ykTp9lyRaAma"
},
"source": [
"model.save(path+name+\".h5\")\n"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "5J4xDoqRUSfS"
},
"source": [
"# save embeddings\n",
"\n",
"# saving embeddings index \n"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "HHlEtipG_Cp0"
},
"source": [
"## Loading models"
]
},
{
"cell_type": "code",
"metadata": {
"id": "fKt8ft1t_Cxx"
},
"source": [
"model = load_model(path+name+\".h5\")\n",
"\n",
"with open(path+tokenizer_filename, 'rb') as file:\n",
" tokenizer = pickle.load(file)\n",
"\n",
"with open(path+encoder_filename, 'rb') as file:\n",
" encoder = pickle.load(file)\n"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "zbS4poso-3k7"
},
"source": [
"## Evaluation"
]
},
{
"cell_type": "code",
"source": [
"df_test = pd.read_csv(test_path, sep=\"\\t\")\n"
],
"metadata": {
"id": "KWORvsadvBbr"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"test_texts = df_test[columnText].tolist()\n",
"test_labels = df_test[columnClass].tolist()\n",
"\n",
"test_sequences = tokenizer.texts_to_sequences(test_texts)\n",
"test_input = sequence.pad_sequences(test_sequences, maxlen=max_sequence_length)\n",
"\n",
"# Get predictions\n",
"test_predictions_probas = model.predict(test_input)\n",
"test_predictions = test_predictions_probas.argmax(axis=-1)"
],
"metadata": {
"id": "Xr0o-0i5t38G"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"\n",
"\n",
"\n",
"test_intent_predictions = encoder.inverse_transform(test_predictions)\n",
"#test_intent_original = encoder.inverse_transform(test_labels)\n",
"\n",
"print('accuracy: ', sum(test_intent_predictions == test_labels) / len(test_labels))\n",
"print(\"Precision, Recall and F1-Score:\\n\\n\", classification_report(test_labels, test_intent_predictions))\n",
"\n"
],
"metadata": {
"id": "lSn8yZ0gt3-d",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "b8077e36-fa15-467d-9bf8-0877982c6f80"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"accuracy: 0.6617672192787558\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, msg_start, len(result))\n",
"/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, msg_start, len(result))\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Precision, Recall and F1-Score:\n",
"\n",
" precision recall f1-score support\n",
"\n",
" Agriculture - Economie rustique 0.40 0.28 0.33 233\n",
" Anatomie 0.79 0.53 0.63 215\n",
" Antiquité 0.47 0.50 0.48 272\n",
" Architecture 0.54 0.47 0.51 278\n",
" Arts et métiers 0.37 0.13 0.20 112\n",
" Beaux-arts 0.49 0.30 0.37 86\n",
" Belles-lettres - Poésie 0.31 0.22 0.26 206\n",
" Blason 0.59 0.48 0.53 108\n",
" Caractères 1.00 0.09 0.16 23\n",
" Chasse 0.66 0.53 0.58 116\n",
" Chimie 0.40 0.31 0.35 104\n",
" Commerce 0.48 0.54 0.51 376\n",
" Droit - Jurisprudence 0.79 0.76 0.77 1284\n",
" Economie domestique 0.20 0.04 0.06 27\n",
" Grammaire 0.43 0.38 0.40 452\n",
" Géographie 0.95 0.94 0.95 2621\n",
" Histoire 0.39 0.53 0.45 616\n",
" Histoire naturelle 0.77 0.82 0.79 963\n",
" Jeu 0.63 0.64 0.64 56\n",
" Marine 0.65 0.71 0.68 415\n",
" Maréchage - Manège 0.85 0.72 0.78 105\n",
" Mathématiques 0.56 0.61 0.58 140\n",
" Mesure 0.33 0.05 0.09 37\n",
" Militaire (Art) - Guerre - Arme 0.57 0.63 0.60 258\n",
" Minéralogie 0.10 0.05 0.06 22\n",
" Monnaie 0.27 0.13 0.17 63\n",
" Musique 0.73 0.53 0.61 137\n",
" Médailles 0.86 0.26 0.40 23\n",
" Médecine - Chirurgie 0.51 0.64 0.57 455\n",
" Métiers 0.53 0.67 0.59 1051\n",
" Pharmacie 0.39 0.14 0.20 65\n",
" Philosophie 0.41 0.27 0.32 94\n",
"Physique - [Sciences physico-mathématiques] 0.52 0.56 0.54 265\n",
" Politique 0.50 0.04 0.08 23\n",
" Pêche 0.75 0.43 0.55 42\n",
" Religion 0.54 0.55 0.55 328\n",
" Spectacle 0.00 0.00 0.00 9\n",
" Superstition 0.86 0.27 0.41 22\n",
"\n",
" accuracy 0.66 11702\n",
" macro avg 0.54 0.41 0.44 11702\n",
" weighted avg 0.66 0.66 0.66 11702\n",
"\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, msg_start, len(result))\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"name = \"test_\"+ name\n",
"\n",
"#classesName = encoder.classes_\n",
"#classes = [str(e) for e in encoder.transform(encoder.classes_)]\n",
"\n",
"report = classification_report(test_labels, test_intent_predictions, output_dict = True)\n",
"\n",
"precision = []\n",
"recall = []\n",
"f1 = []\n",
"support = []\n",
"dff = pd.DataFrame(columns= ['className', 'precision', 'recall', 'f1-score', 'support', 'FP', 'FN', 'TP', 'TN'])\n",
"for c in encoder.classes_:\n",
" precision.append(report[c]['precision'])\n",
" recall.append(report[c]['recall'])\n",
" f1.append(report[c]['f1-score'])\n",
" support.append(report[c]['support'])\n",
"\n",
"accuracy = report['accuracy']\n",
"weighted_avg = report['weighted avg']\n",
"\n",
"\n",
"cnf_matrix = confusion_matrix(test_labels, test_intent_predictions)\n",
"FP = cnf_matrix.sum(axis=0) - np.diag(cnf_matrix)\n",
"FN = cnf_matrix.sum(axis=1) - np.diag(cnf_matrix)\n",
"TP = np.diag(cnf_matrix)\n",
"TN = cnf_matrix.sum() - (FP + FN + TP)\n",
"\n",
"dff['className'] = encoder.classes_\n",
"dff['precision'] = precision\n",
"dff['recall'] = recall\n",
"dff['f1-score'] = f1\n",
"dff['support'] = support\n",
"dff['FP'] = FP\n",
"dff['FN'] = FN\n",
"dff['TP'] = TP\n",
"dff['TN'] = TN\n",
"\n",
"\n",
"\n",
" \n",
"content = name + \"\\n\"\n",
"print(name)\n",
"content += str(weighted_avg) + \"\\n\"\n",
"print(weighted_avg)\n",
"print(accuracy)\n",
"print(dff)\n",
"\n",
"dff.to_csv(path+\"reports/report_\"+name+\".csv\", index=False)\n",
"\n",
"# enregistrer les predictions\n",
"pd.DataFrame({'labels': pd.Series(df_test[columnClass]), 'predictions': pd.Series(test_intent_predictions)}).to_csv(path+\"predictions/predictions_\"+name+\".csv\")\n",
"\n",
"with open(path+\"reports/report_\"+name+\".txt\", 'w') as f:\n",
" f.write(content)\n"
],
"metadata": {
"id": "RQ0LYGuOt4A4",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "bf597b6f-9c83-4fe3-c913-eed91c5610d6"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, msg_start, len(result))\n",
"/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, msg_start, len(result))\n",
"/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, msg_start, len(result))\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"test_cnn_conv1D_egc_glove.6B.100d_s10000\n",
"{'precision': 0.6623969961145181, 'recall': 0.6617672192787558, 'f1-score': 0.6552681646811063, 'support': 11702}\n",
"0.6617672192787558\n",
" className precision ... TP TN\n",
"0 Agriculture - Economie rustique 0.395210 ... 66 11368\n",
"1 Anatomie 0.786207 ... 114 11456\n",
"2 Antiquité 0.470383 ... 135 11278\n",
"3 Architecture 0.543210 ... 132 11313\n",
"4 Arts et métiers 0.365854 ... 15 11564\n",
"5 Beaux-arts 0.490566 ... 26 11589\n",
"6 Belles-lettres - Poésie 0.312500 ... 45 11397\n",
"7 Blason 0.590909 ... 52 11558\n",
"8 Caractères 1.000000 ... 2 11679\n",
"9 Chasse 0.655914 ... 61 11554\n",
"10 Chimie 0.400000 ... 32 11550\n",
"11 Commerce 0.475410 ... 203 11102\n",
"12 Droit - Jurisprudence 0.785084 ... 979 10150\n",
"13 Economie domestique 0.200000 ... 1 11671\n",
"14 Grammaire 0.434010 ... 171 11027\n",
"15 Géographie 0.947469 ... 2471 8944\n",
"16 Histoire 0.389087 ... 328 10571\n",
"17 Histoire naturelle 0.774162 ... 785 10510\n",
"18 Jeu 0.631579 ... 36 11625\n",
"19 Marine 0.647702 ... 296 11126\n",
"20 Maréchage - Manège 0.853933 ... 76 11584\n",
"21 Mathématiques 0.562914 ... 85 11496\n",
"22 Mesure 0.333333 ... 2 11661\n",
"23 Militaire (Art) - Guerre - Arme 0.569930 ... 163 11321\n",
"24 Minéralogie 0.100000 ... 1 11671\n",
"25 Monnaie 0.266667 ... 8 11617\n",
"26 Musique 0.734694 ... 72 11539\n",
"27 Médailles 0.857143 ... 6 11678\n",
"28 Médecine - Chirurgie 0.513228 ... 291 10971\n",
"29 Métiers 0.527820 ... 702 10023\n",
"30 Pharmacie 0.391304 ... 9 11623\n",
"31 Philosophie 0.409836 ... 25 11572\n",
"32 Physique - [Sciences physico-mathématiques] 0.522807 ... 149 11301\n",
"33 Politique 0.500000 ... 1 11678\n",
"34 Pêche 0.750000 ... 18 11654\n",
"35 Religion 0.543807 ... 180 11223\n",
"36 Spectacle 0.000000 ... 0 11693\n",
"37 Superstition 0.857143 ... 6 11679\n",
"\n",
"[38 rows x 9 columns]\n"
]
}
]
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "Lbwg2H8sJRe7"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "4mX5g55AJRhj"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "-D8Gj6kzJRjv"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "Lwz5cO2eJRmD"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "yr_UWq14JRoI"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "WJM6J6_EJRqx"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "CtYR7NTvJRs2"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "Qppm6jATJRvM"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "rK5nK4gyJRx0"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "vSjWwcQKJRz7"
},
"execution_count": null,
"outputs": []
}
]
}