From 21df7dd30824fa80778a6eb1144e64c858aeea6c Mon Sep 17 00:00:00 2001
From: Khalleud <ledk14@gmail.com>
Date: Sat, 11 Sep 2021 14:26:24 +0200
Subject: [PATCH] [ADD] Notebook Bert Fine Tuning
---
BertFineTuning_.ipynb | 1098 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 1098 insertions(+)
create mode 100644 BertFineTuning_.ipynb
diff --git a/BertFineTuning_.ipynb b/BertFineTuning_.ipynb
new file mode 100644
index 0000000..6216aeb
--- /dev/null
+++ b/BertFineTuning_.ipynb
@@ -0,0 +1,1098 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "name": "BertFineTuning_.ipynb",
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "8hzEGHl7gmzk"
+ },
+ "source": [
+ "## Setup GPU"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "dPOU-Efhf4ui",
+ "outputId": "1e3142a8-6351-43f3-9147-68406520b7ee"
+ },
+ "source": [
+ "import torch\n",
+ "\n",
+ "# If there's a GPU available...\n",
+ "if torch.cuda.is_available(): \n",
+ "\n",
+ " # Tell PyTorch to use the GPU. \n",
+ " device = torch.device(\"cuda\")\n",
+ "\n",
+ " print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
+ "\n",
+ " print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
+ "\n",
+ "# If not...\n",
+ "else:\n",
+ " print('No GPU available, using the CPU instead.')\n",
+ " device = torch.device(\"cpu\")"
+ ],
+ "execution_count": 1,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "No GPU available, using the CPU instead.\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Jr-S9yYIgGkA"
+ },
+ "source": [
+ "## Install packages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "pwmZ5bBvgGNh",
+ "outputId": "79c5fb08-a9f4-41bc-eb4d-ab448c5fb4a7"
+ },
+ "source": [
+ "pip install transformers"
+ ],
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.10.0)\n",
+ "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
+ "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (21.0)\n",
+ "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n",
+ "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.0)\n",
+ "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.6.4)\n",
+ "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (5.4.1)\n",
+ "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.45)\n",
+ "Requirement already satisfied: huggingface-hub>=0.0.12 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.16)\n",
+ "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
+ "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
+ "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
+ "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub>=0.0.12->transformers) (3.7.4.3)\n",
+ "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
+ "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.5.0)\n",
+ "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
+ "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
+ "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
+ "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
+ "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
+ "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "VFXEpG00gXkL",
+ "outputId": "2336f39a-78b7-4118-e754-508d876c51f9"
+ },
+ "source": [
+ "pip install sentencepiece"
+ ],
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (0.1.96)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "12SA-qPFgsVo"
+ },
+ "source": [
+ "## Utils functions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "WkIVcabUgxIl"
+ },
+ "source": [
+ "def create_dict(df, classColumnName):\n",
+ " return dict(df[classColumnName].value_counts())\n",
+ "\n",
+ "def remove_weak_classes(df, classColumnName, threshold):\n",
+ "\n",
+ " dictOfClassInstances = create_dict(df,classColumnName)\n",
+ "\n",
+ "\n",
+ " dictionary = {k: v for k, v in dictOfClassInstances.items() if v >= threshold }\n",
+ " keys = [*dictionary]\n",
+ " df_tmp = df[~ df[classColumnName].isin(keys)]\n",
+ " df = pd.concat([df,df_tmp]).drop_duplicates(keep=False)\n",
+ " return df\n",
+ "\n",
+ "\n",
+ "def resample_classes(df, classColumnName, numberOfInstances):\n",
+ " \n",
+ " #random numberOfInstances elements\n",
+ " replace = False # with replacement\n",
+ "\n",
+ " fn = lambda obj: obj.loc[np.random.choice(obj.index, numberOfInstances if len(obj) > numberOfInstances else len(obj), replace),:]\n",
+ " return df.groupby(classColumnName, as_index=False).apply(fn)\n",
+ " "
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "c5QKcXulhNJ-"
+ },
+ "source": [
+ "## Load Data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "vonJ-d4Qg1g5"
+ },
+ "source": [
+ "import pandas as pd \n",
+ "import numpy as np\n",
+ "from sklearn import preprocessing\n",
+ "from sklearn.model_selection import train_test_split"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "ouU5usvXg4PA"
+ },
+ "source": [
+ "dataPath = 'dataframe_with_ensemble_domaine_enccre.csv'\n",
+ "columnText = 'contentWithoutClass'\n",
+ "columnClass = 'ensemble_domaine_enccre'\n",
+ "minOfInstancePerClass = 200\n",
+ "maxOfInstancePerClass = 1500"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "5u1acjunhoxe"
+ },
+ "source": [
+ "df = pd.read_csv(dataPath)\n",
+ "df = remove_weak_classes(df, columnClass, minOfInstancePerClass)\n",
+ "df = resample_classes(df, columnClass, maxOfInstancePerClass)\n",
+ "df = df[df[columnClass] != 'unclassified']"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "zrjZvs2dhzAy"
+ },
+ "source": [
+ "y = df[columnClass]\n",
+ "numberOfClasses = y.nunique()\n",
+ "encoder = preprocessing.LabelEncoder()\n",
+ "y = encoder.fit_transform(y)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "u9AxxaA_h1CM"
+ },
+ "source": [
+ "train_x, test_x, train_y, test_y = train_test_split(df, y, test_size=0.33, random_state=42, stratify = y )\n"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "Xt_PhH_6h1_3"
+ },
+ "source": [
+ "sentences = train_x[columnText].values\n",
+ "labels = train_y.tolist()"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Gs4Agx_5h43M"
+ },
+ "source": [
+ "# Model\n",
+ "## Tokenisation & Input Formatting"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "YZ5PhEYZiCEA"
+ },
+ "source": [
+ "tokeniser_bert = 'bert-base-multilingual-cased'\n",
+ "tokeniser_camembert = 'camembert-base'\n",
+ "\n",
+ "model_bert = \"bert-base-multilingual-cased\"\n",
+ "model_camembert = 'camembert-base'"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "C4bigx_3ibuN",
+ "outputId": "9d54db26-9920-4a92-bb1e-4534f287140f"
+ },
+ "source": [
+ "from transformers import BertTokenizer, CamembertTokenizer\n",
+ "\n",
+ "# Load the BERT tokenizer.\n",
+ "print('Loading BERT tokenizer...')\n",
+ "tokenizer = BertTokenizer.from_pretrained(tokeniser_bert, do_lower_case=True)"
+ ],
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Loading BERT tokenizer...\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "5hNod5X9jDZN",
+ "outputId": "1166b782-d384-4388-de21-21091dc9f925"
+ },
+ "source": [
+ " # Tokenize all of the sentences and map the tokens to thier word IDs.\n",
+ "input_ids = []\n",
+ "\n",
+ "# For every sentence...\n",
+ "for sent in sentences:\n",
+ " # `encode` will:\n",
+ " # (1) Tokenize the sentence.\n",
+ " # (2) Prepend the `[CLS]` token to the start.\n",
+ " # (3) Append the `[SEP]` token to the end.\n",
+ " # (4) Map tokens to their IDs.\n",
+ " encoded_sent = tokenizer.encode(\n",
+ " sent, # Sentence to encode.\n",
+ " add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
+ "\n",
+ " # This function also supports truncation and conversion\n",
+ " # to pytorch tensors, but I need to do padding, so I\n",
+ " # can't use these features.\n",
+ " #max_length = 128, # Truncate all sentences.\n",
+ " #return_tensors = 'pt', # Return pytorch tensors.\n",
+ " )\n",
+ " \n",
+ " # Add the encoded sentence to the list.\n",
+ " input_ids.append(encoded_sent)\n",
+ "\n"
+ ],
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "Token indices sequence length is longer than the specified maximum sequence length for this model (866 > 512). Running this sequence through the model will result in indexing errors\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "W9EWv5JvjGH3",
+ "outputId": "9072122d-3586-40fe-9d75-5b6e9035d6d2"
+ },
+ "source": [
+ "print('Max sentence length: ', max([len(sen) for sen in input_ids])) "
+ ],
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Max sentence length: 3462\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "xh1TQJyvjOx5"
+ },
+ "source": [
+ "max_len = 180\n",
+ "padded = []\n",
+ "for i in input_ids:\n",
+ "\n",
+ " if len(i) > max_len:\n",
+ " padded.extend([i[:max_len]])\n",
+ " else:\n",
+ " padded.extend([i + [0] * (max_len - len(i))])\n",
+ "\n",
+ "\n",
+ "padded = input_ids = np.array(padded)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "ZiwY6gn0jUkD"
+ },
+ "source": [
+ " # Create attention masks\n",
+ "attention_masks = []\n",
+ "\n",
+ "# For each sentence...\n",
+ "for sent in padded:\n",
+ " \n",
+ " # Create the attention mask.\n",
+ " # - If a token ID is 0, then it's padding, set the mask to 0.\n",
+ " # - If a token ID is > 0, then it's a real token, set the mask to 1.\n",
+ " att_mask = [int(token_id > 0) for token_id in sent]\n",
+ " \n",
+ " # Store the attention mask for this sentence.\n",
+ " attention_masks.append(att_mask)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "oBTR5AfAjXJe"
+ },
+ "source": [
+ "# Use 90% for training and 10% for validation.\n",
+ "train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(padded, labels, \n",
+ " random_state=2018, test_size=0.1, stratify = labels )\n",
+ "# Do the same for the masks.\n",
+ "train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,\n",
+ " random_state=2018, test_size=0.1, stratify = labels)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "b9Mw5kq3jhTb"
+ },
+ "source": [
+ "# Convert all inputs and labels into torch tensors, the required datatype \n",
+ "# for my model.\n",
+ "train_inputs = torch.tensor(train_inputs)\n",
+ "validation_inputs = torch.tensor(validation_inputs)\n",
+ "\n",
+ "train_labels = torch.tensor(train_labels)\n",
+ "validation_labels = torch.tensor(validation_labels)\n",
+ "\n",
+ "train_masks = torch.tensor(train_masks)\n",
+ "validation_masks = torch.tensor(validation_masks)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "UfFWzbENjnkw"
+ },
+ "source": [
+ "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
+ "\n",
+ "# The DataLoader needs to know the batch size for training, so I specify it here.\n",
+ "# For fine-tuning BERT on a specific task, the authors recommend a batch size of\n",
+ "# 16 or 32.\n",
+ "\n",
+ "batch_size = 32\n",
+ "\n",
+ "# Create the DataLoader for training set.\n",
+ "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
+ "train_sampler = RandomSampler(train_data)\n",
+ "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
+ "\n",
+ "# Create the DataLoader for validation set.\n",
+ "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
+ "validation_sampler = SequentialSampler(validation_data)\n",
+ "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "x45JNGqhkUn2"
+ },
+ "source": [
+ "## Training"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 463
+ },
+ "id": "C7M2Er1ajsTf",
+ "outputId": "fe4c13b7-5157-49b4-e878-6d7676d4d1a3"
+ },
+ "source": [
+ "from transformers import BertForSequenceClassification, AdamW, BertConfig, CamembertForSequenceClassification\n",
+ "\n",
+ "# Load BertForSequenceClassification, the pretrained BERT model with a single \n",
+ "# linear classification layer on top.\n",
+ "\n",
+ "model = BertForSequenceClassification.from_pretrained(\n",
+ " model_bert, # Use the 12-layer BERT model, with an uncased vocab.\n",
+ " num_labels = numberOfClasses, # The number of output labels--2 for binary classification.\n",
+ " # You can increase this for multi-class tasks. \n",
+ " output_attentions = False, # Whether the model returns attentions weights.\n",
+ " output_hidden_states = False, # Whether the model returns all hidden-states.\n",
+ ")\n",
+ "\n",
+ "# Tell pytorch to run this model on the GPU.\n",
+ "model.cuda()"
+ ],
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertForSequenceClassification: ['cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias']\n",
+ "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+ "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+ "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-multilingual-cased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
+ "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+ ]
+ },
+ {
+ "output_type": "error",
+ "ename": "RuntimeError",
+ "evalue": "ignored",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m<ipython-input-120-80c23ac5f353>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;31m# Tell pytorch to run this model on the GPU.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+ "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mcuda\u001b[0;34m(self, device)\u001b[0m\n\u001b[1;32m 635\u001b[0m \u001b[0mModule\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 636\u001b[0m \"\"\"\n\u001b[0;32m--> 637\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 638\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 639\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mxpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m 528\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 529\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 532\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m 528\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 529\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 532\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m 528\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 529\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 532\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m 550\u001b[0m \u001b[0;31m# `with torch.no_grad():`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 551\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 552\u001b[0;31m \u001b[0mparam_applied\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 553\u001b[0m \u001b[0mshould_use_set_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 554\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mshould_use_set_data\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m 635\u001b[0m \u001b[0mModule\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 636\u001b[0m \"\"\"\n\u001b[0;32m--> 637\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 638\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 639\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mxpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 352.00 MiB (GPU 0; 11.17 GiB total capacity; 10.43 GiB already allocated; 91.81 MiB free; 10.63 GiB reserved in total by PyTorch)"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "xd_cG-8pj4Iw"
+ },
+ "source": [
+ "#Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n",
+ "# I believe the 'W' stands for 'Weight Decay fix\"\n",
+ "optimizer = AdamW(model.parameters(),\n",
+ " lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
+ " eps = 1e-8 # args.adam_epsilon - default is 1e-8.\n",
+ " )"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "65G-uHuLj4_6"
+ },
+ "source": [
+ "from transformers import get_linear_schedule_with_warmup\n",
+ "\n",
+ "# Number of training epochs (authors recommend between 2 and 4)\n",
+ "epochs = 4\n",
+ "\n",
+ "# Total number of training steps is number of batches * number of epochs.\n",
+ "total_steps = len(train_dataloader) * epochs\n",
+ "\n",
+ "# Create the learning rate scheduler.\n",
+ "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
+ " num_warmup_steps = 0, # Default value in run_glue.py\n",
+ " num_training_steps = total_steps)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "lHSOuwcMj9jf"
+ },
+ "source": [
+ "import numpy as np\n",
+ "\n",
+ "# Function to calculate the accuracy of our predictions vs labels\n",
+ "def flat_accuracy(preds, labels):\n",
+ " pred_flat = np.argmax(preds, axis=1).flatten()\n",
+ " labels_flat = labels.flatten()\n",
+ " return np.sum(pred_flat == labels_flat) / len(labels_flat) "
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "Z0S3br-7kASm"
+ },
+ "source": [
+ "import time\n",
+ "import datetime\n",
+ "\n",
+ "def format_time(elapsed):\n",
+ " '''\n",
+ " Takes a time in seconds and returns a string hh:mm:ss\n",
+ " '''\n",
+ " # Round to the nearest second.\n",
+ " elapsed_rounded = int(round((elapsed)))\n",
+ " \n",
+ " # Format as hh:mm:ss\n",
+ " return str(datetime.timedelta(seconds=elapsed_rounded))"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "SbHBbYpwkKaA"
+ },
+ "source": [
+ "import random\n",
+ "\n",
+ "# This training code is based on the `run_glue.py` script here:\n",
+ "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
+ "\n",
+ "\n",
+ "# Set the seed value all over the place to make this reproducible.\n",
+ "seed_val = 42\n",
+ "\n",
+ "random.seed(seed_val)\n",
+ "np.random.seed(seed_val)\n",
+ "torch.manual_seed(seed_val)\n",
+ "torch.cuda.manual_seed_all(seed_val)\n",
+ "\n",
+ "# Store the average loss after each epoch so I can plot them.\n",
+ "loss_values = []\n",
+ "\n",
+ "# For each epoch...\n",
+ "for epoch_i in range(0, epochs):\n",
+ " \n",
+ " # ========================================\n",
+ " # Training\n",
+ " # ========================================\n",
+ " \n",
+ " # Perform one full pass over the training set.\n",
+ "\n",
+ " print(\"\")\n",
+ " print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
+ " print('Training...')\n",
+ "\n",
+ " # Measure how long the training epoch takes.\n",
+ " t0 = time.time()\n",
+ "\n",
+ " # Reset the total loss for this epoch.\n",
+ " total_loss = 0\n",
+ "\n",
+ " # Put the model into training mode.\n",
+ " model.train()\n",
+ "\n",
+ " # For each batch of training data...\n",
+ " for step, batch in enumerate(train_dataloader):\n",
+ "\n",
+ " # Progress update every 40 batches.\n",
+ " if step % 40 == 0 and not step == 0:\n",
+ " # Calculate elapsed time in minutes.\n",
+ " elapsed = format_time(time.time() - t0)\n",
+ " \n",
+ " # Report progress.\n",
+ " print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
+ "\n",
+ " # Unpack this training batch from the dataloader. \n",
+ " #\n",
+ " # As I unpack the batch, I'll also copy each tensor to the GPU using the \n",
+ " # `to` method.\n",
+ " #\n",
+ " # `batch` contains three pytorch tensors:\n",
+ " # [0]: input ids \n",
+ " # [1]: attention masks\n",
+ " # [2]: labels \n",
+ " b_input_ids = batch[0].to(device)\n",
+ " b_input_mask = batch[1].to(device)\n",
+ " b_labels = batch[2].to(device)\n",
+ "\n",
+ " # Always clear any previously calculated gradients before performing a\n",
+ " # backward pass. PyTorch doesn't do this automatically because \n",
+ " # accumulating the gradients is \"convenient while training RNNs\". \n",
+ " # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
+ " model.zero_grad() \n",
+ "\n",
+ " # Perform a forward pass (evaluate the model on this training batch).\n",
+ " # This will return the loss (rather than the model output) because I\n",
+ " # have provided the `labels`.\n",
+ " # The documentation for this `model` function is here: \n",
+ " # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
+ " outputs = model(b_input_ids, \n",
+ " token_type_ids=None, \n",
+ " attention_mask=b_input_mask, \n",
+ " labels=b_labels)\n",
+ " \n",
+ " # The call to `model` always returns a tuple, so I need to pull the \n",
+ " # loss value out of the tuple.\n",
+ " loss = outputs[0]\n",
+ "\n",
+ " # Accumulate the training loss over all of the batches so that I can\n",
+ " # calculate the average loss at the end. `loss` is a Tensor containing a\n",
+ " # single value; the `.item()` function just returns the Python value \n",
+ " # from the tensor.\n",
+ " total_loss += loss.item()\n",
+ "\n",
+ " # Perform a backward pass to calculate the gradients.\n",
+ " loss.backward()\n",
+ "\n",
+ " # Clip the norm of the gradients to 1.0.\n",
+ " # This is to help prevent the \"exploding gradients\" problem.\n",
+ " torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
+ "\n",
+ " # Update parameters and take a step using the computed gradient.\n",
+ " # The optimizer dictates the \"update rule\"--how the parameters are\n",
+ " # modified based on their gradients, the learning rate, etc.\n",
+ " optimizer.step()\n",
+ "\n",
+ " # Update the learning rate.\n",
+ " scheduler.step()\n",
+ "\n",
+ " # Calculate the average loss over the training data.\n",
+ " avg_train_loss = total_loss / len(train_dataloader) \n",
+ " \n",
+ " # Store the loss value for plotting the learning curve.\n",
+ " loss_values.append(avg_train_loss)\n",
+ "\n",
+ " print(\"\")\n",
+ " print(\" Average training loss: {0:.2f}\".format(avg_train_loss))\n",
+ " print(\" Training epoch took: {:}\".format(format_time(time.time() - t0)))\n",
+ " \n",
+ " # ========================================\n",
+ " # Validation\n",
+ " # ========================================\n",
+ " # After the completion of each training epoch, measure the performance on\n",
+ " # the validation set.\n",
+ "\n",
+ " print(\"\")\n",
+ " print(\"Running Validation...\")\n",
+ "\n",
+ " t0 = time.time()\n",
+ "\n",
+ " # Put the model in evaluation mode--the dropout layers behave differently\n",
+ " # during evaluation.\n",
+ " model.eval()\n",
+ "\n",
+ " # Tracking variables \n",
+ " eval_loss, eval_accuracy = 0, 0\n",
+ " nb_eval_steps, nb_eval_examples = 0, 0\n",
+ "\n",
+ " # Evaluate data for one epoch\n",
+ " for batch in validation_dataloader:\n",
+ " \n",
+ " # Add batch to GPU\n",
+ " batch = tuple(t.to(device) for t in batch)\n",
+ " \n",
+ " # Unpack the inputs from dataloader\n",
+ " b_input_ids, b_input_mask, b_labels = batch\n",
+ " \n",
+ " # Telling the model not to compute or store gradients, saving memory and\n",
+ " # speeding up validation\n",
+ " with torch.no_grad(): \n",
+ "\n",
+ " # Forward pass, calculate logit predictions.\n",
+ " # This will return the logits rather than the loss because we have\n",
+ " # not provided labels.\n",
+ " # token_type_ids is the same as the \"segment ids\", which \n",
+ " # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
+ " # The documentation for this `model` function is here: \n",
+ " # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
+ " outputs = model(b_input_ids, \n",
+ " token_type_ids=None, \n",
+ " attention_mask=b_input_mask)\n",
+ " \n",
+ " # Get the \"logits\" output by the model. The \"logits\" are the output\n",
+ " # values prior to applying an activation function like the softmax.\n",
+ " logits = outputs[0]\n",
+ "\n",
+ " # Move logits and labels to CPU\n",
+ " logits = logits.detach().cpu().numpy()\n",
+ " label_ids = b_labels.to('cpu').numpy()\n",
+ " \n",
+ " # Calculate the accuracy for this batch of test sentences.\n",
+ " tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
+ " \n",
+ " # Accumulate the total accuracy.\n",
+ " eval_accuracy += tmp_eval_accuracy\n",
+ "\n",
+ " # Track the number of batches\n",
+ " nb_eval_steps += 1\n",
+ "\n",
+ " # Report the final accuracy for this validation run.\n",
+ " print(\" Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n",
+ " print(\" Validation took: {:}\".format(format_time(time.time() - t0)))\n",
+ "\n",
+ "print(\"\")\n",
+ "print(\"Training complete!\")"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "VJwyfmakkQyj"
+ },
+ "source": [
+ "## Test"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "VAyzmfhZCGZo"
+ },
+ "source": [
+ "sentences_test = test_x[columnText].values\n",
+ "labels_test = test_y.tolist()"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "lZFXr_sdCJcb"
+ },
+ "source": [
+ "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
+ "input_ids_test = []\n",
+ "# For every sentence...\n",
+ "for sent in sentences_test:\n",
+ " # `encode` will:\n",
+ " # (1) Tokenize the sentence.\n",
+ " # (2) Prepend the `[CLS]` token to the start.\n",
+ " # (3) Append the `[SEP]` token to the end.\n",
+ " # (4) Map tokens to their IDs.\n",
+ " encoded_sent = tokenizer.encode(\n",
+ " sent, # Sentence to encode.\n",
+ " add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
+ " )\n",
+ " \n",
+ " input_ids_test.append(encoded_sent)\n",
+ "\n",
+ "# Pad our input tokens\n",
+ "padded_test = []\n",
+ "for i in input_ids_test:\n",
+ "\n",
+ " if len(i) > max_len:\n",
+ " padded_test.extend([i[:max_len]])\n",
+ " else:\n",
+ " padded_test.extend([i + [0] * (max_len - len(i))])\n",
+ "input_ids_test = np.array(padded_test)\n",
+ "\n",
+ "# Create attention masks\n",
+ "attention_masks = []\n",
+ "\n",
+ "# Create a mask of 1s for each token followed by 0s for padding\n",
+ "for seq in input_ids_test:\n",
+ " seq_mask = [float(i>0) for i in seq]\n",
+ " attention_masks.append(seq_mask) \n",
+ "\n",
+ "# Convert to tensors.\n",
+ "prediction_inputs = torch.tensor(input_ids_test)\n",
+ "prediction_masks = torch.tensor(attention_masks)\n",
+ "prediction_labels = torch.tensor(labels_test)\n",
+ "\n",
+ "# Set the batch size. \n",
+ "batch_size = 32 \n",
+ "\n",
+ "# Create the DataLoader.\n",
+ "prediction_data = TensorDataset(prediction_inputs, prediction_masks, prediction_labels)\n",
+ "prediction_sampler = SequentialSampler(prediction_data)\n",
+ "prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "SUDcxi03Cmf-"
+ },
+ "source": [
+ "print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs)))\n",
+ "\n",
+ "# Put model in evaluation mode\n",
+ "model.eval()\n",
+ "\n",
+ "# Tracking variables \n",
+ "predictions_test , true_labels = [], []\n",
+ "\n",
+ "# Predict \n",
+ "for batch in prediction_dataloader:\n",
+ "# Add batch to GPU\n",
+ " batch = tuple(t.to(device) for t in batch)\n",
+ " \n",
+ " # Unpack the inputs from the dataloader\n",
+ " b_input_ids, b_input_mask, b_labels = batch\n",
+ " \n",
+ " # Telling the model not to compute or store gradients, saving memory and \n",
+ " # speeding up prediction\n",
+ " with torch.no_grad():\n",
+ " # Forward pass, calculate logit predictions\n",
+ " outputs = model(b_input_ids, token_type_ids=None, \n",
+ " attention_mask=b_input_mask)\n",
+ "\n",
+ " logits = outputs[0]\n",
+ " #print(logits)\n",
+ "\n",
+ " # Move logits and labels to CPU\n",
+ " logits = logits.detach().cpu().numpy()\n",
+ " label_ids = b_labels.to('cpu').numpy()\n",
+ " #print(logits)\n",
+ " \n",
+ " # Store predictions and true labels\n",
+ " predictions_test.append(logits)\n",
+ " true_labels.append(label_ids)\n",
+ "\n",
+ "print(' DONE.')"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "c3i7szp3Cn5u"
+ },
+ "source": [
+ "from sklearn.metrics import *\n",
+ "\n",
+ "pred_labels = []\n",
+ "\n",
+ "# Evaluate each test batch using many matrics\n",
+ "print('Calculating the matrics for each batch...')\n",
+ "\n",
+ "for i in range(len(true_labels)):\n",
+ " \n",
+ " # The predictions for this batch are a 2-column ndarray (one column for \"0\" \n",
+ " # and one column for \"1\"). Pick the label with the highest value and turn this\n",
+ " # in to a list of 0s and 1s.\n",
+ " pred_labels_i = np.argmax(predictions_test[i], axis=1).flatten()\n",
+ " pred_labels.append(pred_labels_i)\n"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "0bU9-DsBCxSO"
+ },
+ "source": [
+ "pred_labels_ = [item for sublist in pred_labels for item in sublist]\n",
+ "true_labels_ = [item for sublist in true_labels for item in sublist]\n"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ZUM_U2QlC4K5"
+ },
+ "source": [
+ "### Report & Evaluation"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "d5n84N0xCfcU"
+ },
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn.metrics import plot_confusion_matrix\n",
+ "from sklearn.metrics import confusion_matrix\n",
+ "from sklearn.metrics import classification_report\n",
+ "import seaborn as sns"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "v4hXk-KjC-nq"
+ },
+ "source": [
+ "report = classification_report( pred_labels_, true_labels_, output_dict = True)\n",
+ " \n",
+ "accuracy = report['accuracy']\n",
+ "weighted_avg = report['weighted avg']"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "xETMy1L6DAa5"
+ },
+ "source": [
+ "classes = [str(e) for e in encoder.transform(encoder.classes_)]\n",
+ "classesName = encoder.classes_"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "dPjV_5g8DDQy"
+ },
+ "source": [
+ "precision = []\n",
+ "recall = []\n",
+ "f1 = []\n",
+ "support = []\n",
+ "dff = pd.DataFrame(columns= ['className', 'precision', 'recall', 'f1-score', 'support', 'FP', 'FN', 'TP', 'TN'])\n",
+ "for c in classes:\n",
+ " precision.append(report[c]['precision'])\n",
+ " recall.append(report[c]['recall'])\n",
+ " f1.append(report[c]['f1-score'])\n",
+ " support.append(report[c]['support'])\n",
+ "\n",
+ "accuracy = report['accuracy']\n",
+ "weighted_avg = report['weighted avg']\n",
+ "cnf_matrix = confusion_matrix(true_labels_, pred_labels_)\n",
+ "FP = cnf_matrix.sum(axis=0) - np.diag(cnf_matrix)\n",
+ "FN = cnf_matrix.sum(axis=1) - np.diag(cnf_matrix)\n",
+ "TP = np.diag(cnf_matrix)\n",
+ "TN = cnf_matrix.sum() - (FP + FN + TP)\n",
+ "\n",
+ "dff['className'] = classesName\n",
+ "dff['precision'] = precision\n",
+ "dff['recall'] = recall\n",
+ "dff['f1-score'] = f1\n",
+ "dff['support'] = support\n",
+ "dff['FP'] = FP\n",
+ "dff['FN'] = FN\n",
+ "dff['TP'] = TP\n",
+ "dff['TN'] = TN\n",
+ " \n"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "vslzi9bHDKcv"
+ },
+ "source": [
+ "print(weighted_avg)\n",
+ "print(accuracy)\n",
+ "print(dff)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ }
+ ]
+}
\ No newline at end of file
--
GitLab