Skip to content
Snippets Groups Projects
Commit 4731074b authored by Ludovic Moncla's avatar Ludovic Moncla
Browse files

Update Predict.ipynb

parent b7fed500
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
notebooks/Predict.ipynb
+ 205
− 20
View file @ 4731074b
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# BERT Predict classification # BERT Predict classification
## 1. Setup the environment ## 1. Setup the environment
### 1.1 Setup colab environment ### 1.1 Setup colab environment
#### 1.1.1 Install packages #### 1.1.1 Install packages
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
!pip install transformers==4.10.3 !pip install transformers==4.10.3
!pip install sentencepiece !pip install sentencepiece
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
#### 1.1.2 Use more RAM #### 1.1.2 Use more RAM
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
from psutil import virtual_memory from psutil import virtual_memory
ram_gb = virtual_memory().total / 1e9 ram_gb = virtual_memory().total / 1e9
print('Your runtime has {:.1f} gigabytes of available RAM\n'.format(ram_gb)) print('Your runtime has {:.1f} gigabytes of available RAM\n'.format(ram_gb))
if ram_gb < 20: if ram_gb < 20:
print('Not using a high-RAM runtime') print('Not using a high-RAM runtime')
else: else:
print('You are using a high-RAM runtime!') print('You are using a high-RAM runtime!')
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
#### 1.1.3 Mount GoogleDrive #### 1.1.3 Mount GoogleDrive
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
from google.colab import drive from google.colab import drive
drive.mount('/content/drive') drive.mount('/content/drive')
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### 1.2 Setup GPU ### 1.2 Setup GPU
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import torch import torch
# If there's a GPU available... # If there's a GPU available...
if torch.cuda.is_available(): if torch.cuda.is_available():
# Tell PyTorch to use the GPU. # Tell PyTorch to use the GPU.
device = torch.device("cuda") device = torch.device("cuda")
print('There are %d GPU(s) available.' % torch.cuda.device_count()) print('There are %d GPU(s) available.' % torch.cuda.device_count())
print('We will use the GPU:', torch.cuda.get_device_name(0)) print('We will use the GPU:', torch.cuda.get_device_name(0))
# for MacOS # for MacOS
elif torch.backends.mps.is_available() and torch.backends.mps.is_built(): elif torch.backends.mps.is_available() and torch.backends.mps.is_built():
device = torch.device("mps") device = torch.device("mps")
print('We will use the GPU') print('We will use the GPU')
else: else:
device = torch.device("cpu") device = torch.device("cpu")
print('No GPU available, using the CPU instead.') print('No GPU available, using the CPU instead.')
``` ```
%% Output
We will use the GPU
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### 1.3 Import librairies ### 1.3 Import librairies
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import pandas as pd import pandas as pd
import numpy as np import numpy as np
from transformers import BertTokenizer, BertForSequenceClassification, CamembertTokenizer, TextClassificationPipeline from transformers import BertTokenizer, BertForSequenceClassification, CamembertTokenizer, TextClassificationPipeline
from torch.utils.data import TensorDataset, DataLoader, SequentialSampler from torch.utils.data import TensorDataset, DataLoader, SequentialSampler
import pickle import pickle
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## 2. Load Data ## 2. Load Data
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
#!wget https://geode.liris.cnrs.fr/files/datasets/EDdA/Classification/LGE_withContent.tsv #!wget https://geode.liris.cnrs.fr/files/datasets/EDdA/Classification/LGE_withContent.tsv
#!wget https://geode.liris.cnrs.fr/EDdA-Classification/datasets/EDdA_dataset_articles_no_superdomain.tsv #!wget https://geode.liris.cnrs.fr/EDdA-Classification/datasets/EDdA_dataset_articles_no_superdomain.tsv
#!wget https://geode.liris.cnrs.fr/EDdA-Classification/datasets/Parallel_datatset_articles_230215.tsv #!wget https://geode.liris.cnrs.fr/EDdA-Classification/datasets/Parallel_datatset_articles_230215.tsv
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
#drive_path = "drive/MyDrive/Classification-EDdA/" #drive_path = "drive/MyDrive/Classification-EDdA/"
drive_path = "../" drive_path = "../"
#path = "/Users/lmoncla/git/gitlab.liris/GEODE/EDdA/output/" #path = "/Users/lmoncla/git/gitlab.liris/GEODE/EDdA/output/"
path = "/Users/lmoncla/Nextcloud-LIRIS/GEODE/GEODE - Partage consortium/Corpus/LGE/" path = "/Users/lmoncla/Nextcloud-LIRIS/GEODE/GEODE - Partage consortium/Corpus/LGE/"
#filepath = "Parallel_datatset_articles_230215.tsv" #filepath = "Parallel_datatset_articles_230215.tsv"
#filepath = "EDdA_dataset_articles.tsv" #filepath = "EDdA_dataset_articles.tsv"
filepath = 'LGE_dataset_articles_230314.tsv' filepath = 'LGE_dataset_articles_230314.tsv'
corpus = 'lge' corpus = 'lge'
#corpus = '' #corpus = ''
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
df = pd.read_csv(path + filepath, sep="\t") df = pd.read_csv(path + filepath, sep="\t")
df.head() df.head()
``` ```
%% Output
uid lge-volume lge-numero lge-head lge-page lge-id \
0 lge_1_a-0 1 1 A 0 a-0
1 lge_1_a-1 1 2 A 1 a-1
2 lge_1_a-2 1 3 A 4 a-2
3 lge_1_a-3 1 4 A 4 a-3
4 lge_1_a-4 1 5 A 4 a-4
lge-content lge-nbWords
0 A(Ling.). Son vocal et première lettre de notr... 1761.0
1 A(Paléogr.). C’est à l’alphabet phénicien, on ... 839.0
2 A(Log.). Cette voyelle désigne les proposition... 56.0
3 A(Mus.). La lettre a est employée par les musi... 267.0
4 A(Numis.). Dans la numismatique grecque, la le... 67.0
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
dataset = df[corpus+'-content'].values dataset = df[corpus+'-content'].values
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## 3. Load model and predict ## 3. Load model and predict
### 3.1 BERT / CamemBERT ### 3.1 BERT / CamemBERT
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
model_name = "bert-base-multilingual-cased" model_name = "bert-base-multilingual-cased"
#model_name = "camembert-base" #model_name = "camembert-base"
#model_path = path + "models/model_" + model_name + "_s10000.pt" #model_path = path + "models/model_" + model_name + "_s10000.pt"
model_path = drive_path + "models/model_" + model_name + "_s10000_superdomains.pt" model_path = drive_path + "models/model_" + model_name + "_s10000_superdomains.pt"
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def generate_dataloader(tokenizer, sentences, batch_size = 8, max_len = 512): def generate_dataloader(tokenizer, sentences, batch_size = 8, max_len = 512):
# Tokenize all of the sentences and map the tokens to thier word IDs. # Tokenize all of the sentences and map the tokens to thier word IDs.
input_ids_test = [] input_ids_test = []
# For every sentence... # For every sentence...
for sent in sentences: for sent in sentences:
# `encode` will: # `encode` will:
# (1) Tokenize the sentence. # (1) Tokenize the sentence.
# (2) Prepend the `[CLS]` token to the start. # (2) Prepend the `[CLS]` token to the start.
# (3) Append the `[SEP]` token to the end. # (3) Append the `[SEP]` token to the end.
# (4) Map tokens to their IDs. # (4) Map tokens to their IDs.
encoded_sent = tokenizer.encode( encoded_sent = tokenizer.encode(
sent, # Sentence to encode. sent, # Sentence to encode.
add_special_tokens = True, # Add '[CLS]' and '[SEP]' add_special_tokens = True, # Add '[CLS]' and '[SEP]'
# This function also supports truncation and conversion # This function also supports truncation and conversion
# to pytorch tensors, but I need to do padding, so I # to pytorch tensors, but I need to do padding, so I
# can't use these features. # can't use these features.
#max_length = max_len, # Truncate all sentences. #max_length = max_len, # Truncate all sentences.
#return_tensors = 'pt', # Return pytorch tensors. #return_tensors = 'pt', # Return pytorch tensors.
) )
input_ids_test.append(encoded_sent) input_ids_test.append(encoded_sent)
# Pad our input tokens # Pad our input tokens
padded_test = [] padded_test = []
for i in input_ids_test: for i in input_ids_test:
if len(i) > max_len: if len(i) > max_len:
padded_test.extend([i[:max_len]]) padded_test.extend([i[:max_len]])
else: else:
padded_test.extend([i + [0] * (max_len - len(i))]) padded_test.extend([i + [0] * (max_len - len(i))])
input_ids_test = np.array(padded_test) input_ids_test = np.array(padded_test)
# Create attention masks # Create attention masks
attention_masks = [] attention_masks = []
# Create a mask of 1s for each token followed by 0s for padding # Create a mask of 1s for each token followed by 0s for padding
for seq in input_ids_test: for seq in input_ids_test:
seq_mask = [float(i>0) for i in seq] seq_mask = [float(i>0) for i in seq]
attention_masks.append(seq_mask) attention_masks.append(seq_mask)
# Convert to tensors. # Convert to tensors.
inputs = torch.tensor(input_ids_test) inputs = torch.tensor(input_ids_test)
masks = torch.tensor(attention_masks) masks = torch.tensor(attention_masks)
#set batch size #set batch size
# Create the DataLoader. # Create the DataLoader.
data = TensorDataset(inputs, masks) data = TensorDataset(inputs, masks)
prediction_sampler = SequentialSampler(data) prediction_sampler = SequentialSampler(data)
return DataLoader(data, sampler=prediction_sampler, batch_size=batch_size) return DataLoader(data, sampler=prediction_sampler, batch_size=batch_size)
def predict(model, dataloader, device): def predict(model, dataloader, device):
# Put model in evaluation mode # Put model in evaluation mode
model.eval() model.eval()
# Tracking variables # Tracking variables
predictions_test , true_labels = [], [] predictions_test , true_labels = [], []
pred_labels_ = [] pred_labels_ = []
# Predict # Predict
for batch in dataloader: for batch in dataloader:
# Add batch to GPU # Add batch to GPU
batch = tuple(t.to(device) for t in batch) batch = tuple(t.to(device) for t in batch)
# Unpack the inputs from the dataloader # Unpack the inputs from the dataloader
b_input_ids, b_input_mask = batch b_input_ids, b_input_mask = batch
# Telling the model not to compute or store gradients, saving memory and # Telling the model not to compute or store gradients, saving memory and
# speeding up prediction # speeding up prediction
with torch.no_grad(): with torch.no_grad():
# Forward pass, calculate logit predictions # Forward pass, calculate logit predictions
outputs = model(b_input_ids, token_type_ids=None, outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask) attention_mask=b_input_mask)
logits = outputs[0] logits = outputs[0]
#print(logits) #print(logits)
# Move logits and labels to CPU ??? # Move logits and labels to CPU ???
logits = logits.detach().cpu().numpy() logits = logits.detach().cpu().numpy()
#print(logits) #print(logits)
# Store predictions and true labels # Store predictions and true labels
predictions_test.append(logits) predictions_test.append(logits)
pred_labels = [] pred_labels = []
for i in range(len(predictions_test)): for i in range(len(predictions_test)):
# The predictions for this batch are a 2-column ndarray (one column for "0" # The predictions for this batch are a 2-column ndarray (one column for "0"
# and one column for "1"). Pick the label with the highest value and turn this # and one column for "1"). Pick the label with the highest value and turn this
# in to a list of 0s and 1s. # in to a list of 0s and 1s.
pred_labels_i = np.argmax(predictions_test[i], axis=1).flatten() pred_labels_i = np.argmax(predictions_test[i], axis=1).flatten()
pred_labels.append(pred_labels_i) pred_labels.append(pred_labels_i)
pred_labels_ += [item for sublist in pred_labels for item in sublist] pred_labels_ += [item for sublist in pred_labels for item in sublist]
return pred_labels_ return pred_labels_
#https://discuss.huggingface.co/t/i-have-trained-my-classifier-now-how-do-i-do-predictions/3625/3 #https://discuss.huggingface.co/t/i-have-trained-my-classifier-now-how-do-i-do-predictions/3625/3
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
if model_name == 'bert-base-multilingual-cased' : if model_name == 'bert-base-multilingual-cased' :
print('Loading Bert Tokenizer...') print('Loading Bert Tokenizer...')
tokenizer = BertTokenizer.from_pretrained(model_name) tokenizer = BertTokenizer.from_pretrained(model_name)
elif model_name == 'camembert-base': elif model_name == 'camembert-base':
print('Loading Camembert Tokenizer...') print('Loading Camembert Tokenizer...')
tokenizer = CamembertTokenizer.from_pretrained(model_name) tokenizer = CamembertTokenizer.from_pretrained(model_name)
``` ```
%% Output
Loading Bert Tokenizer...
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
https://discuss.huggingface.co/t/an-efficient-way-of-loading-a-model-that-was-saved-with-torch-save/9814 https://discuss.huggingface.co/t/an-efficient-way-of-loading-a-model-that-was-saved-with-torch-save/9814
https://github.com/huggingface/transformers/issues/2094 https://github.com/huggingface/transformers/issues/2094
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
model = BertForSequenceClassification.from_pretrained(model_path).to(device.type) model = BertForSequenceClassification.from_pretrained(model_path).to(device.type)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
#data_loader = generate_dataloader(tokenizer, data) #data_loader = generate_dataloader(tokenizer, data)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
#pred = predict(model, data_loader, device) #pred = predict(model, data_loader, device)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# https://huggingface.co/docs/transformers/main_classes/pipelines # https://huggingface.co/docs/transformers/main_classes/pipelines
def data(): #TODO : def data(): #TODO :
for d in dataset: for d in dataset:
yield f"{d}" yield f"{d}"
pipe = TextClassificationPipeline(model=model, tokenizer=tokenizer, return_all_scores=True, device=device) pipe = TextClassificationPipeline(model=model, tokenizer=tokenizer, return_all_scores=True, device=device)
# https://stackoverflow.com/questions/67849833/how-to-truncate-input-in-the-huggingface-pipeline # https://stackoverflow.com/questions/67849833/how-to-truncate-input-in-the-huggingface-pipeline
tokenizer_kwargs = {'padding':True, 'truncation':True, 'max_length':512} tokenizer_kwargs = {'padding':True, 'truncation':True, 'max_length':512}
``` ```
%% Output
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/transformers/pipelines/text_classification.py:89: UserWarning: `return_all_scores` is now deprecated, if want a similar funcionality use `top_k=None` instead of `return_all_scores=True` or `top_k=1` instead of `return_all_scores=False`.
warnings.warn(
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
pred = [] pred = []
cpt = 0
for out in pipe(data(), **tokenizer_kwargs): for out in pipe(data(), **tokenizer_kwargs):
out = sorted(out, key=lambda d: d['score'], reverse=True) out = sorted(out, key=lambda d: d['score'], reverse=True)
print(int(out[0]['label'][6:]), out[0]['score'], int(out[1]['label'][6:]), out[1]['score'], int(out[2]['label'][6:]), out[2]['score']) # label ### TODO modifier ici #print(int(out[0]['label'][6:]), out[0]['score'], int(out[1]['label'][6:]), out[1]['score'], int(out[2]['label'][6:]), out[2]['score']) # label ### TODO modifier ici
pred.append([int(out[0]['label'][6:]), out[0]['score'], int(out[1]['label'][6:]), out[1]['score'], int(out[2]['label'][6:]), out[2]['score']]) pred.append([int(out[0]['label'][6:]), out[0]['score'], int(out[1]['label'][6:]), out[1]['score'], int(out[2]['label'][6:]), out[2]['score']])
cpt += 1
if cpt == 6:
break
pred = np.array(pred) pred = np.array(pred)
``` ```
%% Output
13 0.9375858902931213 2 0.021192006766796112 7 0.012656938284635544
6 0.9926056861877441 7 0.0029343003407120705 8 0.0010190330212935805
13 0.9823671579360962 2 0.00412388751283288 1 0.0022031590342521667
10 0.9058945775032043 2 0.029459038749337196 7 0.014979560859501362
7 0.9861114025115967 2 0.003949115984141827 6 0.0015271392185240984
4 0.9868664741516113 5 0.002140316180884838 15 0.0018120049498975277
6 0.9541037678718567 7 0.025117166340351105 8 0.00887206755578518
6 0.9981995820999146 7 0.00028012823895551264 8 0.00019026087829843163
6 0.9958584904670715 8 0.0010782132158055902 7 0.000548136536963284
6 0.9979164004325867 7 0.0005610007210634649 9 0.00018632493447512388
6 0.997787356376648 7 0.0003991609555669129 8 0.0002408416330581531
6 0.9979755282402039 7 0.0005005390848964453 8 0.0002189433143939823
11 0.9915592074394226 3 0.00250804889947176 14 0.0010435001458972692
7 0.958525538444519 2 0.011816944926977158 5 0.009215029887855053
8 0.2876076102256775 7 0.2462710738182068 2 0.17002692818641663
8 0.9409826397895813 7 0.03510138392448425 6 0.007040794938802719
8 0.3623795211315155 1 0.3142264485359192 7 0.13734686374664307
7 0.7184596061706543 6 0.11600398272275925 8 0.09759759902954102
7 0.8406069278717041 6 0.12032385170459747 2 0.009349718689918518
7 0.978775143623352 2 0.005065936129540205 4 0.0037283776327967644
6 0.4818583130836487 9 0.22724471986293793 5 0.07886118441820145
6 0.9740952253341675 8 0.015889622271060944 1 0.001933401683345437
---------------------------------------------------------------------------
KeyboardInterrupt Traceback (most recent call last)
/var/folders/qm/v_b1md29221_cnpcxf5qc43c0000gn/T/ipykernel_15176/3568789409.py in <cell line: 3>()
1 pred = []
2
----> 3 for out in pipe(data(), **tokenizer_kwargs):
4 out = sorted(out, key=lambda d: d['score'], reverse=True)
5 print(int(out[0]['label'][6:]), out[0]['score'], int(out[1]['label'][6:]), out[1]['score'], int(out[2]['label'][6:]), out[2]['score']) # label ### TODO modifier ici
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/transformers/pipelines/pt_utils.py in __next__(self)
112
113 # We're out of items within a batch
--> 114 item = next(self.iterator)
115 processed = self.infer(item, **self.params)
116 # We now have a batch of "inferred things".
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/transformers/pipelines/pt_utils.py in __next__(self)
113 # We're out of items within a batch
114 item = next(self.iterator)
--> 115 processed = self.infer(item, **self.params)
116 # We now have a batch of "inferred things".
117 if self.loader_batch_size is not None:
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/transformers/pipelines/base.py in forward(self, model_inputs, **forward_params)
988 with inference_context():
989 model_inputs = self._ensure_tensor_on_device(model_inputs, device=self.device)
--> 990 model_outputs = self._forward(model_inputs, **forward_params)
991 model_outputs = self._ensure_tensor_on_device(model_outputs, device=torch.device("cpu"))
992 else:
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/transformers/pipelines/text_classification.py in _forward(self, model_inputs)
165
166 def _forward(self, model_inputs):
--> 167 return self.model(**model_inputs)
168
169 def postprocess(self, model_outputs, function_to_apply=None, top_k=1, _legacy=True):
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1188 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1189 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1190 return forward_call(*input, **kwargs)
1191 # Do not call functions when jit is used
1192 full_backward_hooks, non_full_backward_hooks = [], []
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/transformers/models/bert/modeling_bert.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
1550 return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1551
-> 1552 outputs = self.bert(
1553 input_ids,
1554 attention_mask=attention_mask,
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1188 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1189 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1190 return forward_call(*input, **kwargs)
1191 # Do not call functions when jit is used
1192 full_backward_hooks, non_full_backward_hooks = [], []
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/transformers/models/bert/modeling_bert.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
985 # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
986 # ourselves in which case we just need to make it broadcastable to all heads.
--> 987 extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape)
988
989 # If a 2D or 3D attention mask is provided for the cross-attention
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/transformers/modeling_utils.py in get_extended_attention_mask(self, attention_mask, input_shape, device, dtype)
789 # effectively the same as removing these entirely.
790 extended_attention_mask = extended_attention_mask.to(dtype=dtype) # fp16 compatibility
--> 791 extended_attention_mask = (1.0 - extended_attention_mask) * torch.finfo(dtype).min
792 return extended_attention_mask
793
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/torch/_tensor.py in wrapped(*args, **kwargs)
37 if has_torch_function(args):
38 return handle_torch_function(wrapped, args, *args, **kwargs)
---> 39 return f(*args, **kwargs)
40 except TypeError:
41 return NotImplemented
/opt/homebrew/Caskroom/miniforge/base/envs/geode-classification-py39/lib/python3.9/site-packages/torch/_tensor.py in __rsub__(self, other)
831 @_handle_torch_function_and_wrap_type_error_to_not_implemented
832 def __rsub__(self, other):
--> 833 return _C._VariableFunctions.rsub(self, other)
834
835 @_handle_torch_function_and_wrap_type_error_to_not_implemented
KeyboardInterrupt:
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# Load label encoder # Load label encoder
#encoder_filename = "models/label_encoder.pkl" #encoder_filename = "models/label_encoder.pkl"
encoder_filename = "models/label_encoder_superdomains.pkl" encoder_filename = "models/label_encoder_superdomains.pkl"
with open(drive_path + encoder_filename, 'rb') as file: with open(drive_path + encoder_filename, 'rb') as file:
encoder = pickle.load(file) encoder = pickle.load(file)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
pred1 = list(encoder.inverse_transform(pred[:,0].astype(int))) pred1 = list(encoder.inverse_transform(pred[:,0].astype(int)))
pred2 = list(encoder.inverse_transform(pred[:,2].astype(int))) pred2 = list(encoder.inverse_transform(pred[:,2].astype(int)))
pred3 = list(encoder.inverse_transform(pred[:,4].astype(int))) pred3 = list(encoder.inverse_transform(pred[:,4].astype(int)))
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
#print(pred1) #print(pred1)
#print(pred[:,1]) #print(pred[:,1])
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
df[corpus+'-superdomainPred1'] = pred1 df[corpus+'-superdomainPred1'] = pred1
df[corpus+'-superdomainProba1'] = pred[:,1] df[corpus+'-superdomainProba1'] = pred[:,1]
df[corpus+'-superdomainPred2'] = pred2 df[corpus+'-superdomainPred2'] = pred2
df[corpus+'-superdomainProba2'] = pred[:,3] df[corpus+'-superdomainProba2'] = pred[:,3]
df[corpus+'-superdomainPred3'] = pred3 df[corpus+'-superdomainPred3'] = pred3
df[corpus+'-superdomainProba3'] = pred[:,5] df[corpus+'-superdomainProba3'] = pred[:,5]
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
df.head(10) df.head(10)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
#df.to_csv(drive_path + "predictions/EDdA_dataset_articles_superdomainBERT_230313.tsv", sep="\t") #df.to_csv(drive_path + "predictions/EDdA_dataset_articles_superdomainBERT_230313.tsv", sep="\t")
df.to_csv(drive_path + "predictions/LGE_dataset_articles_superdomainBERT_230321.tsv", sep="\t", index=False) df.to_csv(drive_path + "predictions/LGE_dataset_articles_superdomainBERT_230321.tsv", sep="\t", index=False)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
#df.drop(columns=['contentLGE', 'contentEDdA'], inplace=True) #df.drop(columns=['contentLGE', 'contentEDdA'], inplace=True)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
df.loc[(df[corpus+'-superdomainProba1'] == 'Géographie')] df.loc[(df[corpus+'-superdomainProba1'] == 'Géographie')]
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
df.shape df.shape
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
``` ```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment