Skip to content
Snippets Groups Projects
Commit 176e6ca5 authored by Jacques Fize's avatar Jacques Fize
Browse files

add binder support

parent 31876184
No related branches found
No related tags found
No related merge requests found
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
!jupyter contrib nbextension install --user
!jupyter nbextension enable --py widgetsnbextension
```
%% Cell type:code id: tags:
``` python
from glob import glob from glob import glob
import pandas as pd import pandas as pd
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from ipywidgets import interact, interactive, fixed, interact_manual from ipywidgets import interact, interactive, fixed, interact_manual
import ipywidgets as widgets import ipywidgets as widgets
from glob import glob from glob import glob
import json import json
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import seaborn as sns import seaborn as sns
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import matplotlib as mpl import matplotlib as mpl
sns.set(style="whitegrid") sns.set(style="whitegrid")
sns.set_context('paper') sns.set_context('paper')
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
base_dir = "FR_RESULTS/" # CHANGE TO "TEXAS_IDF_RESULTS" to plot the results obtained on the "Ile de France + Texas" dataset base_dir = "FR_RESULTS/" # CHANGE TO "TEXAS_IDF_RESULTS" to plot the results obtained on the "Ile de France + Texas" dataset
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
DATASET = "FRANCE" # Change dataset name to IDF_TEXAS if base_dir = "TEXAS_IDF_RESULTS" DATASET = "FRANCE" # Change dataset name to IDF_TEXAS if base_dir = "TEXAS_IDF_RESULTS"
fns = glob(base_dir + "/*.json") fns = glob(base_dir + "/*.json")
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
json.load(open(fns[0])) json.load(open(fns[0]))
``` ```
%% Output %% Output
{'dataset_name': 'FR.txt', {'dataset_name': 'FR.txt',
'rel_code': 'A', 'rel_code': 'A',
'cooc_sample_size': 3, 'cooc_sample_size': 3,
'adj_iteration': 1, 'adj_iteration': 1,
'ngram_size': 4, 'ngram_size': 4,
'tolerance_value': 0.002, 'tolerance_value': 0.002,
'epochs': 100, 'epochs': 100,
'embedding_dim': 256, 'embedding_dim': 256,
'word2vec_iter_nb': 50, 'word2vec_iter_nb': 50,
'index_fn': 'outputs/FR.txt_100_4_0.002__A_index', 'index_fn': 'outputs/FR.txt_100_4_0.002__A_index',
'keras_model_fn': 'outputs/FR.txt_100_4_0.002__A.h5', 'keras_model_fn': 'outputs/FR.txt_100_4_0.002__A.h5',
'train_test_history_fn': 'outputs/FR.txt_100_4_0.002__A.csv'} 'train_test_history_fn': 'outputs/FR.txt_100_4_0.002__A.csv'}
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def decode_fn(filename): def decode_fn(filename):
""" """
Extract data and metadata from an output log file Extract data and metadata from an output log file
""" """
data = json.load(open(filename)) data = json.load(open(filename))
return { return {
"dataset":data["dataset_name"], "dataset":data["dataset_name"],
"epochs":data["epochs"], "epochs":data["epochs"],
"ngram size":data["ngram_size"], "ngram size":data["ngram_size"],
"k":data["tolerance_value"], "k":data["tolerance_value"],
"tuple" : data["rel_code"], "tuple" : data["rel_code"],
"data":pd.read_csv(base_dir + data["train_test_history_fn"].split("/")[-1],index_col=0) "data":pd.read_csv(base_dir + data["train_test_history_fn"].split("/")[-1],index_col=0)
} }
def plot(fig,ax,data,keys,test=False): def plot(fig,ax,data,keys,test=False):
""" """
Generate a plot from an output log Generate a plot from an output log
""" """
for k in keys: for k in keys:
new_k = k new_k = k
label = "{0}-N{1} (Train)".format("LSTM",data["ngram size"]) label = "{0}-N{1} (Train)".format("LSTM",data["ngram size"])
data["data"][new_k].plot(label=label,legend=True,ax=ax)#+" Train",legend=True)#,color=color[k],ax=ax) data["data"][new_k].plot(label=label,legend=True,ax=ax)#+" Train",legend=True)#,color=color[k],ax=ax)
if test: if test:
new_k="val_"+new_k new_k="val_"+new_k
label = "{0}-N{1} (Test)".format("LSTM",data["ngram size"]) label = "{0}-N{1} (Test)".format("LSTM",data["ngram size"])
data["data"][new_k].plot(label=label,legend=True,ax=ax,linestyle='--')#+" Train",legend=True)#,color=color[k],ax=ax) data["data"][new_k].plot(label=label,legend=True,ax=ax,linestyle='--')#+" Train",legend=True)#,color=color[k],ax=ax)
ax.legend(bbox_to_anchor=(1.04,1), prop={'size': 15}) ax.legend(bbox_to_anchor=(1.04,1), prop={'size': 15})
ax.set_ylim((0,1)) ax.set_ylim((0,1))
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
kvalue = widgets.Dropdown( kvalue = widgets.Dropdown(
options=[0.001,0.002], options=[0.001,0.002],
value=0.002, value=0.002,
description='K-value:', description='K-value:',
disabled=False, disabled=False,
) )
spatrel = widgets.Dropdown( spatrel = widgets.Dropdown(
options=["I","A","AI","AIC","AC","IC","C"], options=["I","A","AI","AIC","AC","IC","C"],
value="I", value="I",
description='Spatial Relation Used:', description='Spatial Relation Used:',
disabled=False, disabled=False,
) )
def foo(kvalue,spatrel): def foo(kvalue,spatrel):
keys=["Output_LON_accuracy_at_k_lon"]#,"Output_LAT_accuracy_at_k"] keys=["Output_LON_accuracy_at_k_lon"]#,"Output_LAT_accuracy_at_k"]
fig, (ax,ax2) = plt.subplots(2,figsize=(10,10)) fig, (ax,ax2) = plt.subplots(2,figsize=(10,10))
for fn in fns: for fn in fns:
data = decode_fn(fn) data = decode_fn(fn)
if data["tuple"] == spatrel and data["k"] == kvalue: if data["tuple"] == spatrel and data["k"] == kvalue:
plot(fig,ax,data,keys,test=True) plot(fig,ax,data,keys,test=True)
ax.set_title("Longitude") ax.set_title("Longitude")
keys=["Output_LAT_accuracy_at_k_lat"] keys=["Output_LAT_accuracy_at_k_lat"]
for fn in fns: for fn in fns:
data = decode_fn(fn) data = decode_fn(fn)
if data["tuple"] == spatrel and data["k"] == kvalue: if data["tuple"] == spatrel and data["k"] == kvalue:
plot(fig,ax2,data,keys,test=True) plot(fig,ax2,data,keys,test=True)
ax2.set_title("Latitude") ax2.set_title("Latitude")
fig.suptitle("LSTM - accuracy@100km - 4-grams - Relation used {0}".format(spatrel),y=0.95,weight="bold") fig.suptitle("LSTM - accuracy@100km - 4-grams - Relation used {0}".format(spatrel),y=0.95,weight="bold")
#plt.savefig("{2}_LSTM_{0}_{1}.png".format(kvalue,spatrel,DATASET),bbox_inches = 'tight') #plt.savefig("{2}_LSTM_{0}_{1}.png".format(kvalue,spatrel,DATASET),bbox_inches = 'tight')
interact(foo,kvalue=kvalue,spatrel=spatrel) interact(foo,kvalue=kvalue,spatrel=spatrel)
``` ```
%% Output %% Output
<function __main__.foo(kvalue, spatrel)> <function __main__.foo(kvalue, spatrel)>
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
data = decode_fn(fns[0]) data = decode_fn(fns[0])
df = data["data"] df = data["data"]
df["Relation(s) Used"] = data["tuple"] df["Relation(s) Used"] = data["tuple"]
df["epochs"] = np.arange(100) df["epochs"] = np.arange(100)
for fn in fns[1:]: for fn in fns[1:]:
data = decode_fn(fn) data = decode_fn(fn)
new_df = data["data"] new_df = data["data"]
new_df["epochs"] = np.arange(100) new_df["epochs"] = np.arange(100)
new_df["Relation(s) Used"] = data["tuple"] new_df["Relation(s) Used"] = data["tuple"]
df = pd.concat((df,new_df)) df = pd.concat((df,new_df))
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
df.sort_values(by="Relation(s) Used") df.sort_values(by="Relation(s) Used")
``` ```
%% Output %% Output
val_loss val_Output_LON_loss val_Output_LAT_loss \ val_loss val_Output_LON_loss val_Output_LAT_loss \
0 0.000115 0.000045 0.000070 0 0.000115 0.000045 0.000070
72 0.000058 0.000022 0.000036 72 0.000058 0.000022 0.000036
71 0.000058 0.000022 0.000036 71 0.000058 0.000022 0.000036
70 0.000056 0.000022 0.000034 70 0.000056 0.000022 0.000034
69 0.000058 0.000022 0.000036 69 0.000058 0.000022 0.000036
.. ... ... ... .. ... ... ...
27 0.000084 0.000030 0.000054 27 0.000084 0.000030 0.000054
26 0.000084 0.000032 0.000051 26 0.000084 0.000032 0.000051
24 0.000084 0.000027 0.000057 24 0.000084 0.000027 0.000057
35 0.000090 0.000032 0.000058 35 0.000090 0.000032 0.000058
99 0.000096 0.000031 0.000065 99 0.000096 0.000031 0.000065
val_Output_LON_accuracy_at_k_lon val_Output_LAT_accuracy_at_k_lat \ val_Output_LON_accuracy_at_k_lon val_Output_LAT_accuracy_at_k_lat \
0 0.295377 0.509287 0 0.295377 0.509287
72 0.584888 0.779450 72 0.584888 0.779450
71 0.587047 0.775844 71 0.587047 0.775844
70 0.591935 0.784623 70 0.591935 0.784623
69 0.586314 0.775825 69 0.586314 0.775825
.. ... ... .. ... ...
27 0.699088 0.819236 27 0.699088 0.819236
26 0.696359 0.823050 26 0.696359 0.823050
24 0.673946 0.816693 24 0.673946 0.816693
35 0.713059 0.829505 35 0.713059 0.829505
99 0.740629 0.841372 99 0.740629 0.841372
loss Output_LON_loss Output_LAT_loss Output_LON_accuracy_at_k_lon \ loss Output_LON_loss Output_LAT_loss Output_LON_accuracy_at_k_lon \
0 0.000381 0.000188 0.000193 0.273675 0 0.000381 0.000188 0.000193 0.273675
72 0.000004 0.000002 0.000003 0.957315 72 0.000004 0.000002 0.000003 0.957315
71 0.000004 0.000002 0.000003 0.954727 71 0.000004 0.000002 0.000003 0.954727
70 0.000005 0.000002 0.000003 0.953021 70 0.000005 0.000002 0.000003 0.953021
69 0.000005 0.000002 0.000003 0.950501 69 0.000005 0.000002 0.000003 0.950501
.. ... ... ... ... .. ... ... ... ...
27 0.000030 0.000013 0.000018 0.726226 27 0.000030 0.000013 0.000018 0.726226
26 0.000034 0.000014 0.000021 0.709992 26 0.000034 0.000014 0.000021 0.709992
24 0.000032 0.000013 0.000019 0.718581 24 0.000032 0.000013 0.000019 0.718581
35 0.000024 0.000010 0.000014 0.760739 35 0.000024 0.000010 0.000014 0.760739
99 0.000010 0.000005 0.000006 0.872880 99 0.000010 0.000005 0.000006 0.872880
Output_LAT_accuracy_at_k_lat Relation(s) Used epochs Output_LAT_accuracy_at_k_lat Relation(s) Used epochs
0 0.425386 A 0 0 0.425386 A 0
72 0.995521 A 72 72 0.995521 A 72
71 0.995618 A 71 71 0.995618 A 71
70 0.994972 A 70 70 0.994972 A 70
69 0.994548 A 69 69 0.994548 A 69
.. ... ... ... .. ... ... ...
27 0.883270 IC 27 27 0.883270 IC 27
26 0.865662 IC 26 26 0.865662 IC 26
24 0.875147 IC 24 24 0.875147 IC 24
35 0.911580 IC 35 35 0.911580 IC 35
99 0.972612 IC 99 99 0.972612 IC 99
[700 rows x 12 columns] [700 rows x 12 columns]
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
df = df[df.epochs.isin(list(range(0,100,5)))] df = df[df.epochs.isin(list(range(0,100,5)))]
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
g = sns.FacetGrid(df, col="Relation(s) Used", col_wrap=4, height=3, ylim=(0, 1),xlim=(0,100)) g = sns.FacetGrid(df, col="Relation(s) Used", col_wrap=4, height=3, ylim=(0, 1),xlim=(0,100))
g.map(sns.lineplot,"epochs", "Output_LON_accuracy_at_k_lon", color="#e74c3c", ci=None,label="Longitude Train accuracy",marker="o",markersize=7); g.map(sns.lineplot,"epochs", "Output_LON_accuracy_at_k_lon", color="#e74c3c", ci=None,label="Longitude Train accuracy",marker="o",markersize=7);
g.map(sns.lineplot,"epochs", "val_Output_LON_accuracy_at_k_lon", color="#e74c3c", ci=None,label="Longitude Test accuracy",marker="^",markersize=7); g.map(sns.lineplot,"epochs", "val_Output_LON_accuracy_at_k_lon", color="#e74c3c", ci=None,label="Longitude Test accuracy",marker="^",markersize=7);
g.map(sns.lineplot,"epochs", "Output_LAT_accuracy_at_k_lat", color="#2980b9", ci=None,label="Latitude Train accuracy",marker="o",markersize=7); g.map(sns.lineplot,"epochs", "Output_LAT_accuracy_at_k_lat", color="#2980b9", ci=None,label="Latitude Train accuracy",marker="o",markersize=7);
g.map(sns.lineplot,"epochs", "val_Output_LAT_accuracy_at_k_lat", color="#2980b9", ci=None,label="Latitude Test accuracy",marker="^",markersize=7); g.map(sns.lineplot,"epochs", "val_Output_LAT_accuracy_at_k_lat", color="#2980b9", ci=None,label="Latitude Test accuracy",marker="^",markersize=7);
g.set(xlabel='Epochs', ylabel='Accuracy@100km') g.set(xlabel='Epochs', ylabel='Accuracy@100km')
g.set(xticks=np.arange(0,100,10)) g.set(xticks=np.arange(0,100,10))
g.add_legend(bbox_to_anchor=(0.80, 0.20),fontsize="large",title="Legend",title_fontsize="40",frameon=True) g.add_legend(bbox_to_anchor=(0.80, 0.20),fontsize="large",title="Legend",title_fontsize="40",frameon=True)
#g.fig.suptitle('Longitude Prediction (Accuracy@100km)',position=(0.58,.09),fontsize=15) #g.fig.suptitle('Longitude Prediction (Accuracy@100km)',position=(0.58,.09),fontsize=15)
g.fig.subplots_adjust(top=.9) g.fig.subplots_adjust(top=.9)
plt.setp(g._legend.get_title(), fontsize=15) plt.setp(g._legend.get_title(), fontsize=15)
#plt.savefig("{0}_100km.pdf".format(DATASET),bbox_inches = 'tight') #plt.savefig("{0}_100km.pdf".format(DATASET),bbox_inches = 'tight')
``` ```
%% Output %% Output
[None, None] [None, None]
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
``` ```
......
[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/git/https%3A%2F%2Fgitlab.liris.cnrs.fr%2Fjfize%2Fnotebooks-toponym-geocoding.git/master)
# Notebooks Toponym Geocoding # Notebooks Toponym Geocoding
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment