Skip to content
Snippets Groups Projects
Commit 7cfaeaa8 authored by Fize Jacques's avatar Fize Jacques
Browse files

Init Depo

parent ac497563
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
FBScocialCorrectedness.ipynb 0 → 100644
+ 1170
− 0
View file @ 7cfaeaa8
source diff could not be displayed: it is too large. Options to address this: view the blob.
FB_Analysis2.ipynb 0 → 100644
+ 287
− 0
View file @ 7cfaeaa8
source diff could not be displayed: it is too large. Options to address this: view the blob.
draw_graph.ipynb 0 → 100644
+ 411
− 0
View file @ 7cfaeaa8
This diff is collapsed.
evalNE_script.py 0 → 100644
+ 70
− 0
View file @ 7cfaeaa8
from evalne.evaluation.evaluator import LPEvaluator
from evalne.evaluation.split import EvalSplit as LPEvalSplit
from evalne.evaluation.score import Scoresheet
from evalne.utils import preprocess as pp
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("edgelist_graph_filename")
args = parser.parse_args()#("data/fb_country_country_sample_6_size1000.txt".split())
# Load and preprocess the network
G = pp.load_graph(args.edgelist_graph_filename,directed=True)
G, _ = pp.prep_graph(G,maincc=True)
# Create an evaluator and generate train/test edge split
traintest_split = LPEvalSplit()
traintest_split.compute_splits(G,split_alg="spanning_tree",train_frac=0.8,fe_ratio=1)
nee = LPEvaluator(traintest_split)
# Create a Scoresheet to store the results
scoresheet = Scoresheet()
# Set the baselines
methods = ['random_prediction',
'common_neighbours',
'jaccard_coefficient',
"adamic_adar_index",
"preferential_attachment",
"resource_allocation_index"
]
# Evaluate baselines
for method in methods:
result = nee.evaluate_baseline(method=method)
scoresheet.log_results(result)
try:
# Check if OpenNE is installed
import openne
# Set embedding methods from OpenNE
methods = "node2vec hope-opne gf sdne deepWalk line grarep".split() #lap-opne
commands = [
"python -m openne --method node2vec --graph-format edgelist --epochs 100 --number-walks 10 --walk-length 80 --window-size 10",
#"python -m openne --method lap --epochs 100",
"python -m openne --method hope --epochs 100",
"python -m openne --method gf --epochs 100",
"python -m openne --method sdne --epochs 100 --encoder-list [1024,128] --beta 5 --bs 500",
"python -m openne --method deepWalk --graph-format edgelist --epochs 100 --number-walks 10 --walk-length 80 --window-size 10",
"python -m openne --method line --graph-format edgelist --epochs 10",
"python -m openne --method grarep --epochs 100"
]
edge_emb = ['average', 'hadamard']
# Evaluate embedding methods
for i in range(len(methods)):
command = commands[i] + " --input {} --output {} --representation-size {}"
results = nee.evaluate_cmd(method_name=methods[i], method_type='ne', command=command,
edge_embedding_methods=edge_emb, input_delim=' ', output_delim=' ')
scoresheet.log_results(results)
except ImportError:
print("The OpenNE library is not installed. Reporting results only for the baselines...")
pass
# Get output
scoresheet.print_tabular()
scoresheet.write_all(args.edgelist_graph_filename+"_results_lp")
\ No newline at end of file
explore_fb_dataset.ipynb 0 → 100644
+ 640
− 0
View file @ 7cfaeaa8
%% Cell type:code id: tags:
``` python
import networkx as nx
import pandas as pd
import numpy as np
```
%% Cell type:code id: tags:
``` python
df = pd.read_csv("data/gadm1_nuts2_gadm1_nuts2_aug2020.tsv",sep="\t")
```
%% Cell type:code id: tags:
``` python
df
```
%% Output
user_loc fr_loc scaled_sci
0 ABW ABW 13297827
1 ABW AGO1 29
2 ABW AGO10 54
3 ABW AGO11 41
4 ABW AGO12 42
... ... ... ...
5978020 ZWE9 ZWE5 491990
5978021 ZWE9 ZWE6 524119
5978022 ZWE9 ZWE7 929477
5978023 ZWE9 ZWE8 966771
5978024 ZWE9 ZWE9 16951824
[5978025 rows x 3 columns]
%% Cell type:code id: tags:
``` python
level_df = pd.read_csv("data/gadm1_nuts2_levels.csv")
level_df
```
%% Output
key level
0 ASM country
1 AND country
2 ATG country
3 ABW country
4 BHS country
... ... ...
2440 TR42 nuts2
2441 TR51 nuts2
2442 TR52 nuts2
2443 TR10 nuts2
2444 TR32 nuts2
[2445 rows x 2 columns]
%% Cell type:code id: tags:
``` python
level_df[level_df.key == "ABW"]
```
%% Output
key level
3 ABW country
%% Cell type:code id: tags:
``` python
import geopandas as gpd
gdf = gpd.read_file("data/ref-nuts-2021-10m/NUTS_RG_10M_2021_3035_LEVL_2.geojson")
gdf
```
%% Output
id NUTS_ID LEVL_CODE CNTR_CODE NAME_LATN NUTS_NAME \
0 DE50 DE50 2 DE Bremen Bremen
1 DE60 DE60 2 DE Hamburg Hamburg
2 DE71 DE71 2 DE Darmstadt Darmstadt
3 DE72 DE72 2 DE Gießen Gießen
4 DE73 DE73 2 DE Kassel Kassel
.. ... ... ... ... ... ...
329 HR06 HR06 2 HR Sjeverna Hrvatska Sjeverna Hrvatska
330 NO02 NO02 2 NO Innlandet Innlandet
331 NO06 NO06 2 NO Trøndelag Trøndelag
332 NO07 NO07 2 NO Nord-Norge Nord-Norge
333 NO08 NO08 2 NO Oslo og Viken Oslo og Viken
MOUNT_TYPE URBN_TYPE COAST_TYPE FID \
0 0 NaN NaN DE50
1 0 NaN NaN DE60
2 0 NaN NaN DE71
3 0 NaN NaN DE72
4 0 NaN NaN DE73
.. ... ... ... ...
329 0 0.0 0.0 HR06
330 0 NaN NaN NO02
331 0 NaN NaN NO06
332 0 NaN NaN NO07
333 0 0.0 0.0 NO08
geometry
0 MULTIPOLYGON (((4248229.070 3323043.884, 42345...
1 MULTIPOLYGON (((4336708.861 3376535.119, 43414...
2 POLYGON ((4253056.068 3043343.224, 4257541.935...
3 POLYGON ((4248924.963 3092384.236, 4258523.883...
4 POLYGON ((4299188.570 3163540.672, 4298283.911...
.. ...
329 POLYGON ((4885838.460 2569452.540, 4878828.590...
330 POLYGON ((4438332.480 4360687.112, 4440904.728...
331 MULTIPOLYGON (((4414585.206 4664076.456, 44179...
332 MULTIPOLYGON (((5073773.420 5207018.495, 50676...
333 POLYGON ((4424393.606 4083582.648, 4429455.154...
[334 rows x 11 columns]
%% Cell type:code id: tags:
``` python
gdf.info()
```
%% Output
<class 'geopandas.geodataframe.GeoDataFrame'>
RangeIndex: 334 entries, 0 to 333
Data columns (total 11 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 id 334 non-null object
1 NUTS_ID 334 non-null object
2 LEVL_CODE 334 non-null int64
3 CNTR_CODE 334 non-null object
4 NAME_LATN 334 non-null object
5 NUTS_NAME 334 non-null object
6 MOUNT_TYPE 334 non-null int64
7 URBN_TYPE 7 non-null float64
8 COAST_TYPE 7 non-null float64
9 FID 334 non-null object
10 geometry 334 non-null geometry
dtypes: float64(2), geometry(1), int64(2), object(6)
memory usage: 28.8+ KB
%% Cell type:code id: tags:
``` python
```
generate_graph.ipynb 0 → 100644
+ 108
− 0
View file @ 7cfaeaa8
%% Cell type:code id: tags:
``` python
%load_ext autoreload
%autoreload 2
```
%% Cell type:code id: tags:
``` python
import numpy as np
```
%% Cell type:code id: tags:
``` python
from utils import load_country_country_data,sample_with_pandas,to_edgelist
from joblib import dump
from sklearn.preprocessing import LabelEncoder
df = load_country_country_data("data/country_country_aug2020.tsv")
df["norm_scaled_sci"] = df.scaled_sci/df.scaled_sci.sum()
```
%% Cell type:code id: tags:
``` python
!mkdir data/graph_second_s
```
%% Cell type:code id: tags:
``` python
encoder = LabelEncoder()
encoder.fit(np.concatenate((df.user_loc.values,df.fr_loc.values)))
for i in range(10):
for size in [50,100,200,500,1000]:
test = sample_with_pandas(df,size)
to_edgelist(test,encoder,weight=True).to_csv("data/graph_second_s/fb_country_country_sample_{0}_size{1}.txt".format(i,size),index=False,header= False,sep=",")
```
%% Cell type:code id: tags:
``` python
dump(encoder,"data/graph_second_s/encoder.joblib")
```
%% Output
['data/graph_second_s/encoder.joblib']
%% Cell type:code id: tags:
``` python
```
helpers.py 0 → 100644
+ 19
− 0
View file @ 7cfaeaa8
import pandas as pd
import geopandas as gpd
import numpy as np
import networkx as nx
def get_centroid(gdf,key_id):
gdf["centroid_"] = gdf.centroid.apply(lambda coord: [coord.x,coord.y])
return dict(gdf[(key_id + " centroid_").split()].values)
def get_labels(gdf,key_id,key_label):
return dict(gdf[(key_id + " " + key_label).split()].values)
def to_networkx(df,coords_dict, labels_dict):
nodelist = df.user_loc.unique().tolist()
nodelist.extend(df.fr_loc.unique().tolist())
G = nx.from_pandas_edgelist(df,source = "user_loc",target="sfr_loc",edge_attr="weight")
return G
\ No newline at end of file
nx2gt.py 0 → 100644
+ 119
− 0
View file @ 7cfaeaa8
# code from http://bbengfort.github.io/snippets/2016/06/23/graph-tool-from-networkx.html
import networkx as nx
import graph_tool as gt
def get_prop_type(value, key=None):
"""
Performs typing and value conversion for the graph_tool PropertyMap class.
If a key is provided, it also ensures the key is in a format that can be
used with the PropertyMap. Returns a tuple, (type name, value, key)
"""
# Deal with the value
if isinstance(value, bool):
tname = 'bool'
elif isinstance(value, int):
tname = 'float'
value = float(value)
elif isinstance(value, float):
tname = 'float'
elif isinstance(value, dict):
tname = 'object'
else:
tname = 'string'
value = str(value)
return tname, value, key
def nx2gt(nxG):
"""
Converts a networkx graph to a graph-tool graph.
"""
# Phase 0: Create a directed or undirected graph-tool Graph
gtG = gt.Graph(directed=nxG.is_directed())
# Add the Graph properties as "internal properties"
for key, value in nxG.graph.items():
# Convert the value and key into a type for graph-tool
tname, value, key = get_prop_type(value, key)
prop = gtG.new_graph_property(tname) # Create the PropertyMap
gtG.graph_properties[key] = prop # Set the PropertyMap
gtG.graph_properties[key] = value # Set the actual value
# Phase 1: Add the vertex and edge property maps
# Go through all nodes and edges and add seen properties
# Add the node properties first
nprops = set() # cache keys to only add properties once
for node, data in nxG.nodes(data=True):
# Go through all the properties if not seen and add them.
for key, val in data.items():
if key in nprops:
continue # Skip properties already added
# Convert the value and key into a type for graph-tool
tname, _, key = get_prop_type(val, key)
prop = gtG.new_vertex_property(tname) # Create the PropertyMap
gtG.vertex_properties[key] = prop # Set the PropertyMap
# Add the key to the already seen properties
nprops.add(key)
# Also add the node id: in NetworkX a node can be any hashable type, but
# in graph-tool node are defined as indices. So we capture any strings
# in a special PropertyMap called 'id' -- modify as needed!
gtG.vertex_properties['id'] = gtG.new_vertex_property('string')
# Add the edge properties second
eprops = set() # cache keys to only add properties once
for src, dst, data in nxG.edges(data=True):
# Go through all the edge properties if not seen and add them.
for key, val in data.items():
if key in eprops:
continue # Skip properties already added
# Convert the value and key into a type for graph-tool
tname, _, key = get_prop_type(val, key)
prop = gtG.new_edge_property(tname) # Create the PropertyMap
gtG.edge_properties[key] = prop # Set the PropertyMap
# Add the key to the already seen properties
eprops.add(key)
# Phase 2: Actually add all the nodes and vertices with their properties
# Add the nodes
vertices = {} # vertex mapping for tracking edges later
for node, data in nxG.nodes(data=True):
# Create the vertex and annotate for our edges later
v = gtG.add_vertex()
vertices[node] = v
# Set the vertex properties, not forgetting the id property
data['id'] = str(node)
for key, value in data.items():
gtG.vp[key][v] = value # vp is short for vertex_properties
# Add the edges
for src, dst, data in nxG.edges(data=True):
# Look up the vertex structs from our vertices mapping and add edge.
e = gtG.add_edge(vertices[src], vertices[dst])
# Add the edge properties
for key, value in data.items():
gtG.ep[key][e] = value # ep is short for edge_properties
# Done, finally!
return gtG
\ No newline at end of file
utils.py 0 → 100644
+ 43
− 0
View file @ 7cfaeaa8
import pandas as pd
from sklearn.preprocessing import LabelEncoder
import numpy as np
def load_country_country_data(filename,self_link=False):
df = pd.read_csv(filename,sep="\t")
df = df[(~df.user_loc.isna()) & (~df.fr_loc.isna())]
ign = ["CW","XK"] # No coords for these two countries ... got to investigate!
df = df[(~df.user_loc.isin(ign)) & (~df.fr_loc.isin(ign))]
if not self_link:
mask = df.apply(lambda x:False if x.user_loc ==x.fr_loc else True,axis=1)
df = df[mask]
return df
def sample_with_pandas(df,N):
"""
Return a sample of the avalaible connection using Pandas Dataframe.sample() method
Parameters
----------
df : pd.Dataframe
input
Returns
-------
pd.DataFrame
Selected edges
"""
if not "norm_scaled_sci" in df.columns.values:
df["norm_scaled_sci"] = df.scaled_sci/df.scaled_sci.sum()
return df.sample(n=N,weights="norm_scaled_sci").rename(columns={"norm_scaled_sci":"weight"})
def to_edgelist(sample,encoder,weight=False):
new_df = sample.copy()
new_df["fr_loc"] = encoder.transform(new_df.fr_loc.values)
new_df["user_loc"] = encoder.transform(new_df.user_loc.values)
del new_df["scaled_sci"]
if not weight:
del new_df["weight"]
return new_df
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment