diff --git a/training_bertFineTuning.py b/training_bertFineTuning.py index 662647865ff5e42ae8db2b9343c83299336d3a7d..afd9e7d7563ff1b490ea97cb6695d759798e4b87 100644 --- a/training_bertFineTuning.py +++ b/training_bertFineTuning.py @@ -13,7 +13,7 @@ import random import os import argparse import configparser - +import csv @@ -464,7 +464,7 @@ if __name__ == "__main__": epochs = int(config.get('model','epochs')) - df = pd.read_csv(INPUT_DATASET, sep="\t") + df = pd.read_csv(INPUT_DATASET, sep="\t", quoting=csv.QUOTE_NONE) df = remove_weak_classes(df, columnClass, minOfInstancePerClass) df = resample_classes(df, columnClass, maxOfInstancePerClass) #df = df[df[columnClass] != 'unclassified']