Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
O
outillage
Manage
Activity
Members
Labels
Plan
Issues
0
Issue boards
Milestones
Wiki
Code
Merge requests
0
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Alice Brenon
outillage
Commits
d7e8544f
Commit
d7e8544f
authored
1 year ago
by
Alice Brenon
Browse files
Options
Downloads
Patches
Plain Diff
Encoder seems to belong to the BERT model
parent
0d657426
No related branches found
Branches containing commit
No related tags found
No related merge requests found
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
scripts/ML/BERT/Base.py
+14
-14
14 additions, 14 deletions
scripts/ML/BERT/Base.py
scripts/ML/BERT/Classifier.py
+2
-2
2 additions, 2 deletions
scripts/ML/BERT/Classifier.py
with
16 additions
and
16 deletions
scripts/ML/BERT/Base.py
+
14
−
14
View file @
d7e8544f
...
...
@@ -12,20 +12,6 @@ def get_device():
print
(
'
No GPU available, using the CPU instead.
'
)
return
torch
.
device
(
"
cpu
"
)
def
get_encoder
(
root_path
,
create_from
=
None
):
path
=
f
"
{
root_path
}
/label_encoder.pkl
"
if
os
.
path
.
isfile
(
path
):
with
open
(
path
,
'
rb
'
)
as
pickled
:
return
pickle
.
load
(
pickled
)
elif
create_from
is
not
None
:
encoder
=
preprocessing
.
LabelEncoder
()
encoder
.
fit
(
create_from
)
with
open
(
path
,
'
wb
'
)
as
file
:
pickle
.
dump
(
encoder
,
file
)
return
encoder
else
:
raise
FileNotFoundError
(
path
)
def
loader
(
f
):
def
wrapped
(
*
args
,
**
kwargs
):
name
=
f
.
__name__
.
replace
(
'
_init_
'
,
''
)
...
...
@@ -63,6 +49,20 @@ class BERT:
bert
=
BertForSequenceClassification
.
from_pretrained
(
self
.
root_path
)
self
.
model
=
bert
.
to
(
self
.
device
.
type
)
@loader
def
_init_encoder
(
self
,
create_from
=
None
):
path
=
f
"
{
self
.
root_path
}
/label_encoder.pkl
"
if
os
.
path
.
isfile
(
path
):
with
open
(
path
,
'
rb
'
)
as
pickled
:
self
.
encoder
=
pickle
.
load
(
pickled
)
elif
create_from
is
not
None
:
self
.
encoder
=
preprocessing
.
LabelEncoder
()
self
.
encoder
.
fit
(
create_from
)
with
open
(
path
,
'
wb
'
)
as
file
:
pickle
.
dump
(
self
.
encoder
,
file
)
else
:
raise
FileNotFoundError
(
path
)
def
import_data
(
self
,
data
):
return
map
(
lambda
d
:
d
.
to
(
self
.
device
),
data
)
...
...
This diff is collapsed.
Click to expand it.
scripts/ML/BERT/Classifier.py
+
2
−
2
View file @
d7e8544f
from
BERT.Base
import
BERT
,
get_encoder
from
BERT.Base
import
BERT
import
numpy
from
tqdm
import
tqdm
from
transformers
import
TextClassificationPipeline
...
...
@@ -19,7 +19,7 @@ class Classifier(BERT):
def
__init__
(
self
,
root_path
):
BERT
.
__init__
(
self
,
root_path
)
self
.
_init_pipe
()
self
.
encoder
=
ge
t_encoder
(
root_path
)
self
.
_ini
t_encoder
()
def
_init_pipe
(
self
):
self
.
pipe
=
TextClassificationPipeline
(
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment