Skip to content
Snippets Groups Projects
Commit 41bc7131 authored by Ludovic Moncla's avatar Ludovic Moncla
Browse files

Update Predict.ipynb

parent a6f959d2
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 1 addition and 29 deletions
notebooks/Predict.ipynb
+ 1
− 29
View file @ 41bc7131
%% Cell type:markdown id: tags:
# BERT Predict classification
## 1. Setup the environment
### 1.1 Setup colab environment
#### 1.1.1 Install packages
%% Cell type:code id: tags:
``` python
!pip install transformers==4.10.3
!pip install sentencepiece
```
%% Cell type:markdown id: tags:
#### 1.1.2 Use more RAM
%% Cell type:code id: tags:
``` python
from psutil import virtual_memory
ram_gb = virtual_memory().total / 1e9
print('Your runtime has {:.1f} gigabytes of available RAM\n'.format(ram_gb))
if ram_gb < 20:
print('Not using a high-RAM runtime')
else:
print('You are using a high-RAM runtime!')
```
%% Cell type:markdown id: tags:
#### 1.1.3 Mount GoogleDrive
%% Cell type:code id: tags:
``` python
from google.colab import drive
drive.mount('/content/drive')
```
%% Cell type:markdown id: tags:
### 1.2 Setup GPU
%% Cell type:code id: tags:
``` python
import torch
# If there's a GPU available...
if torch.cuda.is_available():
# Tell PyTorch to use the GPU.
device = torch.device("cuda")
print('There are %d GPU(s) available.' % torch.cuda.device_count())
print('We will use the GPU:', torch.cuda.get_device_name(0))
# for MacOS
elif torch.backends.mps.is_available() and torch.backends.mps.is_built():
device = torch.device("mps")
print('We will use the GPU')
else:
device = torch.device("cpu")
print('No GPU available, using the CPU instead.')
```
%% Output
We will use the GPU
%% Cell type:markdown id: tags:
### 1.3 Import librairies
%% Cell type:code id: tags:
``` python
import pandas as pd
import numpy as np
from transformers import BertTokenizer, BertForSequenceClassification, CamembertTokenizer, CamembertForSequenceClassification
from torch.utils.data import TensorDataset, DataLoader, SequentialSampler
```
%% Cell type:markdown id: tags:
## 2. Load Data
%% Cell type:code id: tags:
``` python
#!wget https://geode.liris.cnrs.fr/files/datasets/EDdA/Classification/LGE_withContent.tsv
#!wget https://geode.liris.cnrs.fr/EDdA-Classification/datasets/EDdA_dataset_articles_no_superdomain.tsv
#!wget https://geode.liris.cnrs.fr/EDdA-Classification/datasets/Parallel_datatset_articles_230215.tsv
```
%% Cell type:code id: tags:
``` python
#drive_path = "drive/MyDrive/Classification-EDdA/"
drive_path = "../"
#path = "/Users/lmoncla/git/gitlab.liris/GEODE/EDdA/output/"
path = "/Users/lmoncla/git/gitlab.liris/GEODE/LGE/output/"
#filepath = "Parallel_datatset_articles_230215.tsv"
#filepath = "EDdA_dataset_articles.tsv"
filepath = "LGE_dataset_articles_230314.tsv"
corpus = 'lge'
#corpus = ''
```
%% Cell type:code id: tags:
``` python
df = pd.read_csv(path + filepath, sep="\t")
df.head()
```
%% Output
uid lge-volume lge-numero lge-head lge-page lge-id \
0 lge_1_a-0 1 1 A 0 a-0
1 lge_1_a-1 1 2 A 1 a-1
2 lge_1_a-2 1 3 A 4 a-2
3 lge_1_a-3 1 4 A 4 a-3
4 lge_1_a-4 1 5 A 4 a-4
lge-content lge-nbWords
0 A(Ling.). Son vocal et première lettre de notr... 1761.0
1 A(Paléogr.). C’est à l’alphabet phénicien, on ... 839.0
2 A(Log.). Cette voyelle désigne les proposition... 56.0
3 A(Mus.). La lettre a est employée par les musi... 267.0
4 A(Numis.). Dans la numismatique grecque, la le... 67.0
%% Cell type:code id: tags:
``` python
data = df[corpus+'-content'].values
```
%% Cell type:markdown id: tags:
## 3. Load model and predict
### 3.1 BERT / CamemBERT
%% Cell type:code id: tags:
``` python
model_name = "bert-base-multilingual-cased"
#model_name = "camembert-base"
#model_path = path + "models/model_" + model_name + "_s10000.pt"
model_path = drive_path + "models/model_" + model_name + "_s10000_superdomains.pt"
```
%% Cell type:code id: tags:
``` python
def generate_dataloader(tokenizer, sentences, batch_size = 8, max_len = 512):
# Tokenize all of the sentences and map the tokens to thier word IDs.
input_ids_test = []
# For every sentence...
for sent in sentences:
# `encode` will:
# (1) Tokenize the sentence.
# (2) Prepend the `[CLS]` token to the start.
# (3) Append the `[SEP]` token to the end.
# (4) Map tokens to their IDs.
encoded_sent = tokenizer.encode(
sent, # Sentence to encode.
add_special_tokens = True, # Add '[CLS]' and '[SEP]'
# This function also supports truncation and conversion
# to pytorch tensors, but I need to do padding, so I
# can't use these features.
#max_length = max_len, # Truncate all sentences.
#return_tensors = 'pt', # Return pytorch tensors.
)
input_ids_test.append(encoded_sent)
# Pad our input tokens
padded_test = []
for i in input_ids_test:
if len(i) > max_len:
padded_test.extend([i[:max_len]])
else:
padded_test.extend([i + [0] * (max_len - len(i))])
input_ids_test = np.array(padded_test)
# Create attention masks
attention_masks = []
# Create a mask of 1s for each token followed by 0s for padding
for seq in input_ids_test:
seq_mask = [float(i>0) for i in seq]
attention_masks.append(seq_mask)
# Convert to tensors.
inputs = torch.tensor(input_ids_test)
masks = torch.tensor(attention_masks)
#set batch size
# Create the DataLoader.
data = TensorDataset(inputs, masks)
prediction_sampler = SequentialSampler(data)
return DataLoader(data, sampler=prediction_sampler, batch_size=batch_size)
def predict(model, dataloader, device):
# Put model in evaluation mode
model.eval()
# Tracking variables
predictions_test , true_labels = [], []
pred_labels_ = []
# Predict
for batch in dataloader:
# Add batch to GPU
batch = tuple(t.to(device) for t in batch)
# Unpack the inputs from the dataloader
b_input_ids, b_input_mask = batch
# Telling the model not to compute or store gradients, saving memory and
# speeding up prediction
with torch.no_grad():
# Forward pass, calculate logit predictions
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
logits = outputs[0]
#print(logits)
# Move logits and labels to CPU ???
logits = logits.detach().cpu().numpy()
#print(logits)
# Store predictions and true labels
predictions_test.append(logits)
pred_labels = []
for i in range(len(predictions_test)):
# The predictions for this batch are a 2-column ndarray (one column for "0"
# and one column for "1"). Pick the label with the highest value and turn this
# in to a list of 0s and 1s.
pred_labels_i = np.argmax(predictions_test[i], axis=1).flatten()
pred_labels.append(pred_labels_i)
pred_labels_ += [item for sublist in pred_labels for item in sublist]
return pred_labels_
```
%% Cell type:code id: tags:
``` python
if model_name == 'bert-base-multilingual-cased' :
print('Loading Bert Tokenizer...')
tokenizer = BertTokenizer.from_pretrained(model_name)
elif model_name == 'camembert-base':
print('Loading Camembert Tokenizer...')
tokenizer = CamembertTokenizer.from_pretrained(model_name)
```
%% Output
Loading Bert Tokenizer...
%% Cell type:code id: tags:
``` python
data_loader = generate_dataloader(tokenizer, data)
```
%% Output
Token indices sequence length is longer than the specified maximum sequence length for this model (3408 > 512). Running this sequence through the model will result in indexing errors
%% Cell type:markdown id: tags:
https://discuss.huggingface.co/t/an-efficient-way-of-loading-a-model-that-was-saved-with-torch-save/9814
https://github.com/huggingface/transformers/issues/2094
%% Cell type:code id: tags:
``` python
#model = torch.load(model_path, map_location=torch.device('mps'))
#model.load_state_dict(torch.load(model_path, map_location=torch.device('mps')))
#model = BertForSequenceClassification.from_pretrained(model_path).to("cuda")
model = BertForSequenceClassification.from_pretrained(model_path).to("mps")
```
%% Cell type:code id: tags:
``` python
pred = predict(model, data_loader, device)
```
%% Cell type:code id: tags:
``` python
pred
```
%% Output
[13,
6,
13,
10,
7,
4,
6,
6,
6,
6,
6,
6,
11,
7,
8,
8,
8,
7,
7,
7,
6,
6,
7,
7,
7,
7,
7,
6,
8,
6,
6,
6,
4,
8,
7,
6,
6,
6,
6,
6,
7,
7,
7,
7,
7,
16,
7,
10,
7,
7,
7,
7,
6,
11,
3,
9,
7,
4,
6,
7,
14,
1,
8,
6,
8,
7,
5,
7,
14,
6,
3,
16,
9,
2,
1,
1,
7,
7,
5,
6,
7,
8,
7,
8,
0,
9,
14,
6,
8,
6,
7,
6,
9,
8,
8,
6,
7,
7,
5,
5,
8,
5,
5,
5,
5,
6,
7,
7,
7,
7,
7,
9,
7,
7,
7,
8,
6,
6,
7,
7,
4,
7,
7,
7,
7,
4,
0,
4,
4,
0,
8,
9,
1,
1,
6,
7,
1,
9,
5,
7,
5,
8,
2,
6,
7,
8,
5,
7,
6,
7,
3,
7,
7,
7,
2,
7,
2,
8,
7,
7,
6,
7,
6,
7,
6,
7,
7,
6,
7,
7,
7,
1,
11,
1,
1,
7,
9,
7,
7,
7,
7,
7,
9,
7,
7,
7,
6,
7,
10,
6,
16,
12,
9,
7,
7,
7,
8,
6,
7,
3,
6,
7,
7,
6,
6,
7,
6,
7,
7,
7,
6,
7,
7,
6,
1,
2,
2,
16,
2,
9,
11,
16,
7,
7,
7,
6,
8,
7,
7,
7,
7,
7,
1,
7,
7,
6,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
8,
7,
7,
7,
7,
7,
7,
13,
6,
7,
7,
7,
7,
5,
8,
9,
11,
8,
7,
11,
9,
7,
7,
7,
7,
7,
8,
7,
13,
8,
7,
12,
6,
7,
5,
8,
11,
8,
14,
2,
11,
1,
7,
10,
11,
8,
7,
6,
6,
7,
16,
7,
6,
7,
7,
1,
8,
10,
7,
7,
8,
1,
1,
7,
7,
8,
9,
13,
8,
16,
7,
6,
8,
7,
7,
7,
6,
16,
13,
6,
7,
7,
5,
6,
7,
8,
7,
6,
6,
6,
6,
6,
6,
11,
8,
7,
7,
6,
8,
6,
6,
6,
11,
1,
6,
11,
14,
6,
10,
6,
6,
8,
5,
7,
7,
7,
16,
7,
7,
13,
7,
6,
7,
6,
7,
7,
8,
9,
13,
7,
7,
8,
5,
7,
8,
3,
14,
8,
14,
8,
7,
5,
6,
8,
8,
8,
8,
8,
9,
7,
7,
3,
6,
7,
7,
5,
6,
6,
5,
16,
7,
7,
7,
6,
9,
6,
16,
6,
7,
5,
6,
8,
11,
7,
7,
6,
6,
5,
2,
7,
8,
6,
13,
11,
14,
7,
8,
16,
7,
7,
7,
8,
9,
0,
2,
6,
8,
3,
6,
1,
6,
6,
6,
16,
7,
3,
16,
6,
6,
6,
13,
5,
7,
9,
7,
2,
6,
6,
6,
7,
13,
6,
14,
6,
7,
7,
7,
5,
5,
6,
7,
6,
8,
9,
9,
7,
7,
5,
7,
11,
7,
4,
6,
9,
7,
7,
3,
6,
12,
9,
7,
1,
7,
7,
7,
7,
8,
6,
7,
7,
8,
13,
7,
7,
7,
7,
6,
7,
6,
7,
7,
7,
7,
6,
7,
13,
7,
6,
6,
7,
7,
7,
8,
7,
1,
2,
7,
7,
7,
6,
5,
9,
7,
2,
6,
3,
4,
6,
16,
5,
5,
5,
5,
5,
5,
6,
8,
8,
8,
13,
5,
5,
5,
1,
8,
7,
2,
14,
8,
11,
8,
7,
16,
7,
7,
7,
7,
16,
7,
7,
16,
7,
7,
16,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
7,
11,
8,
6,
8,
7,
6,
6,
7,
7,
7,
12,
8,
11,
7,
7,
8,
10,
14,
7,
6,
7,
14,
7,
5,
7,
0,
5,
9,
7,
7,
1,
0,
8,
8,
9,
9,
3,
6,
13,
6,
5,
4,
6,
8,
8,
1,
8,
7,
8,
3,
8,
8,
8,
0,
6,
9,
6,
8,
7,
7,
7,
14,
5,
5,
1,
1,
12,
8,
11,
11,
7,
13,
16,
13,
14,
14,
11,
14,
11,
14,
16,
7,
7,
5,
5,
13,
11,
16,
7,
13,
14,
14,
13,
8,
7,
7,
10,
9,
4,
8,
2,
9,
8,
8,
3,
5,
13,
5,
5,
8,
8,
6,
6,
6,
7,
6,
8,
8,
13,
6,
7,
6,
7,
7,
7,
7,
7,
7,
7,
6,
8,
9,
7,
7,
7,
6,
8,
13,
7,
13,
7,
6,
7,
7,
7,
7,
7,
5,
1,
7,
1,
7,
6,
6,
8,
8,
7,
6,
8,
6,
8,
8,
8,
8,
8,
7,
8,
7,
8,
8,
8,
8,
6,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
8,
11,
8,
8,
8,
8,
8,
8,
8,
6,
8,
8,
8,
8,
8,
8,
3,
8,
8,
8,
8,
8,
6,
8,
8,
8,
11,
8,
8,
8,
8,
8,
8,
8,
6,
8,
8,
8,
8,
7,
8,
5,
6,
6,
11,
8,
8,
7,
7,
8,
8,
6,
8,
8,
8,
13,
7,
7,
13,
7,
7,
8,
8,
8,
6,
7,
7,
9,
7,
7,
7,
10,
9,
10,
14,
3,
14,
14,
9,
16,
5,
7,
13,
8,
13,
5,
5,
5,
5,
13,
16,
5,
13,
2,
11,
8,
10,
7,
1,
14,
14,
10,
9,
5,
8,
8,
4,
2,
7,
13,
8,
8,
8,
6,
1,
8,
7,
0,
6,
9,
2,
1,
8,
11,
12,
9,
10,
7,
13,
11,
13,
1,
5,
10,
10,
10,
10,
2,
9,
3,
9,
6,
1,
13,
11,
11,
11,
1,
1,
13,
3,
1,
9,
6,
12,
7,
3,
8,
12,
12,
12,
12,
8,
0,
3,
7,
7,
3,
9,
9,
9,
14,
14,
8,
5,
6,
7,
5,
5,
13,
5,
5,
5,
16,
14,
11,
8,
9,
11,
11,
11,
8,
11,
11,
11,
11,
11,
8,
8,
12,
8,
8,
8,
8,
11,
8,
11,
8,
8,
6,
8,
8,
8,
6,
7,
13,
...]
%% Cell type:code id: tags:
``` python
import pickle
#encoder_filename = "models/label_encoder.pkl"
encoder_filename = "models/label_encoder_superdomains.pkl"
with open(drive_path + encoder_filename, 'rb') as file:
encoder = pickle.load(file)
```
%% Output
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/sklearn/base.py:329: UserWarning: Trying to unpickle estimator LabelEncoder from version 1.0.2 when using version 1.1.3. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:
https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations
warnings.warn(
%% Cell type:code id: tags:
``` python
p2 = list(encoder.inverse_transform(pred))
```
%% Cell type:code id: tags:
``` python
df[corpus+'-superdomainBert'] = p2
```
%% Cell type:code id: tags:
``` python
df[df.numero == 2835][corpus+'-content'].values
```
%% Output
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
/var/folders/qm/v_b1md29221_cnpcxf5qc43c0000gn/T/ipykernel_3552/1621721301.py in <cell line: 1>()
----> 1 df[df.numero == 2835][corpus+'-content'].values
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/pandas/core/generic.py in __getattr__(self, name)
5900 ):
5901 return self[name]
-> 5902 return object.__getattribute__(self, name)
5903
5904 def __setattr__(self, name: str, value) -> None:
AttributeError: 'DataFrame' object has no attribute 'numero'
%% Cell type:code id: tags:
``` python
df.head(10)
```
%% Output
uid lge-volume lge-numero lge-head lge-page lge-id \
0 lge_1_a-0 1 1 A 0 a-0
1 lge_1_a-1 1 2 A 1 a-1
2 lge_1_a-2 1 3 A 4 a-2
3 lge_1_a-3 1 4 A 4 a-3
4 lge_1_a-4 1 5 A 4 a-4
5 lge_1_aa-0 1 6 AA 4 aa-0
6 lge_1_aa-1 1 7 AA 4 aa-1
7 lge_1_aa-2 1 8 AA 5 aa-2
8 lge_1_aa-3 1 9 AA 5 aa-3
9 lge_1_aa-4 1 10 AA 5 aa-4
lge-content lge-nbWords \
0 A(Ling.). Son vocal et première lettre de notr... 1761.0
1 A(Paléogr.). C’est à l’alphabet phénicien, on ... 839.0
2 A(Log.). Cette voyelle désigne les proposition... 56.0
3 A(Mus.). La lettre a est employée par les musi... 267.0
4 A(Numis.). Dans la numismatique grecque, la le... 67.0
5 AA. Ces deux lettres désignent l’atelier monét... 14.0
6 AA. Nom de plusieurs cours d’eau de l’Europe o... 75.0
7 AA. Rivière de France, prend sa source aux Tro... 165.0
8 AA. Rivière de Hollande, affluent de la Dommel... 17.0
9 AA. Nom de deux fleuves de la Russie. Le premi... 71.0
lge-superdomainBert
0 Philosophie
1 Géographie
2 Philosophie
3 Musique
4 Histoire
5 Commerce
6 Géographie
7 Géographie
8 Géographie
9 Géographie
%% Cell type:code id: tags:
``` python
#df.to_csv(drive_path + "predictions/EDdA_dataset_articles_superdomainBERT_230313.tsv", sep="\t")
df.to_csv(drive_path + "predictions/LGE_dataset_articles_superdomainBERT_230314.tsv", sep="\t", index=False)
```
%% Cell type:code id: tags:
``` python
#df.drop(columns=['contentLGE', 'contentEDdA'], inplace=True)
```
%% Cell type:code id: tags:
``` python
df.loc[(df[corpus+'-superdomainBert'] == 'Géographie')]
```
%% Output
uid lge-volume lge-numero lge-head lge-page \
1 lge_1_a-1 1 2 A 1
6 lge_1_aa-1 1 7 AA 4
7 lge_1_aa-2 1 8 AA 5
8 lge_1_aa-3 1 9 AA 5
9 lge_1_aa-4 1 10 AA 5
... ... ... ... ... ...
134800 lge_31_zvornix-0 31 7757 ZVORNIX 1370
134801 lge_31_zweibrücken-0 31 7758 ZWEIBRÜCKEN 1370
134803 lge_31_zwickau-0 31 7760 ZWICKAU 1370
134806 lge_31_zwolle-0 31 7763 ZWOLLE 1371
134819 lge_31_zyrmi-0 31 7776 ZYRMI 1372
lge-id lge-content \
1 a-1 A(Paléogr.). C’est à l’alphabet phénicien, on ...
6 aa-1 AA. Nom de plusieurs cours d’eau de l’Europe o...
7 aa-2 AA. Rivière de France, prend sa source aux Tro...
8 aa-3 AA. Rivière de Hollande, affluent de la Dommel...
9 aa-4 AA. Nom de deux fleuves de la Russie. Le premi...
... ... ...
134800 zvornix-0 ZVORNIX. Ville de Bosnie, sur la r. g. de la D...
134801 zweibrücken-0 ZWEIBRÜCKEN. Ville de Bavière (V. Deux-Ponts).\n
134803 zwickau-0 ZWICKAU. Ville de Saxe, ch.-l. d’un cercle, su...
134806 zwolle-0 ZWOLLE. Ville des Pays-Bas, ch.-l. de la prov....
134819 zyrmi-0 ZYRMI. Ville du Soudan. Ancienne capitale du p...
lge-nbWords lge-superdomainBert
1 839.0 Géographie
6 75.0 Géographie
7 165.0 Géographie
8 17.0 Géographie
9 71.0 Géographie
... ... ...
134800 27.0 Géographie
134801 6.0 Géographie
134803 92.0 Géographie
134806 115.0 Géographie
134819 16.0 Géographie
[50917 rows x 9 columns]
%% Cell type:code id: tags:
``` python
df.shape
```
%% Output
(134820, 9)
%% Cell type:code id: tags:
``` python
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment