Update Normclass2graph.ipynb

cb9be46f · Ludovic Moncla · fc57841e · cb9be46f
Commit cb9be46f authored 2 years ago by Ludovic Moncla
--- a/notebooks/Normclass2graph.ipynb
+++ b/notebooks/Normclass2graph.ipynb
@@ -9,7 +9,7 @@
    },
    {
      "cell_type": "code",
-      "execution_count": 7,
+      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
@@ -19,7 +19,7 @@
    },
    {
      "cell_type": "code",
-      "execution_count": 79,
+      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
@@ -28,27 +28,16 @@
    },
    {
      "cell_type": "code",
-      "execution_count": 80,
+      "execution_count": null,
      "metadata": {},
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "2908"
-            ]
-          },
-          "execution_count": 80,
-          "metadata": {},
-          "output_type": "execute_result"
-        }
-      ],
+      "outputs": [],
      "source": [
        "len(df['edda_class'].unique())"
      ]
    },
    {
      "cell_type": "code",
-      "execution_count": 83,
+      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
@@ -57,27 +46,16 @@
    },
    {
      "cell_type": "code",
-      "execution_count": 84,
+      "execution_count": null,
      "metadata": {},
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "2905"
-            ]
-          },
-          "execution_count": 84,
-          "metadata": {},
-          "output_type": "execute_result"
-        }
-      ],
+      "outputs": [],
      "source": [
        "len(df['edda_class'].unique())"
      ]
    },
    {
      "cell_type": "code",
-      "execution_count": 85,
+      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
@@ -86,42 +64,18 @@
    },
    {
      "cell_type": "code",
-      "execution_count": 86,
+      "execution_count": null,
      "metadata": {},
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "2904"
-            ]
-          },
-          "execution_count": 86,
-          "metadata": {},
-          "output_type": "execute_result"
-        }
-      ],
+      "outputs": [],
      "source": [
        "len(df['edda_class'].unique())"
      ]
    },
    {
      "cell_type": "code",
-      "execution_count": 87,
+      "execution_count": null,
      "metadata": {},
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "array(['Grammaire', 'Ecrivains modernes', 'Calendrier Julien', ...,\n",
-              "       'Jeux militaires françois', 'Gravure antique sur métal',\n",
-              "       'Inscription | Médailles | Poésie'], dtype=object)"
-            ]
-          },
-          "execution_count": 87,
-          "metadata": {},
-          "output_type": "execute_result"
-        }
-      ],
+      "outputs": [],
      "source": [
        "normclasses = df['edda_class'].unique()\n",
        "normclasses"
@@ -129,7 +83,7 @@
    },
    {
      "cell_type": "code",
-      "execution_count": 144,
+      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
@@ -138,55 +92,33 @@
    },
    {
      "cell_type": "code",
-      "execution_count": 145,
+      "execution_count": null,
      "metadata": {},
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "edda_class\n",
-              "Abus des langues                     1\n",
-              "Accord de sons                       1\n",
-              "Acoustique                           6\n",
-              "Agonistique                          1\n",
-              "Agriculture                        127\n",
-              "                                  ... \n",
-              "terme usité parmi les Maréchaux      1\n",
-              "vaisselle d'étain                    1\n",
-              "Ébénisterie                          3\n",
-              "Ébénisterie | Tapisserie             1\n",
-              "Économie rustique                    3\n",
-              "Length: 2904, dtype: int64"
-            ]
-          },
-          "execution_count": 145,
-          "metadata": {},
-          "output_type": "execute_result"
-        }
-      ],
+      "outputs": [],
      "source": [
        "df_group"
      ]
    },
    {
      "cell_type": "code",
-      "execution_count": 151,
+      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "d = {}\n",
        "for normclass, freq in df_group.items():\n",
-        "    #print(normclass, freq)\n",
+        "    print(normclass, freq)\n",
        "\n",
        "    try:        \n",
        "        n = [x.strip() for x in normclass.split('|')] # corrige le probleme du normclass 'Géographie moderne  |'\n",
        "        \n",
-        "        for i in range(1,len(n)):\n",
-        "            t = (n[0],n[i])\n",
-        "            if t not in d:\n",
-        "                d[t] = freq\n",
-        "            else:\n",
-        "                d[t] += freq\n",
+        "        for i in range(0,len(n)):\n",
+        "            for j in range(i+1,len(n)):\n",
+        "                t = (n[i],n[j])\n",
+        "                if t not in d:\n",
+        "                    d[t] = freq\n",
+        "                else:\n",
+        "                    d[t] += freq\n",
        "\n",
        "    except AttributeError:\n",
        "        print(normclass)"
@@ -194,31 +126,20 @@
    },
    {
      "cell_type": "code",
-      "execution_count": 152,
+      "execution_count": null,
      "metadata": {},
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "1257"
-            ]
-          },
-          "execution_count": 152,
-          "metadata": {},
-          "output_type": "execute_result"
-        }
-      ],
+      "outputs": [],
      "source": [
        "len(d.keys())"
      ]
    },
    {
      "cell_type": "code",
-      "execution_count": 162,
+      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
-        "G = nx.DiGraph()\n",
+        "G = nx.Graph()\n",
        "content = 'node1,node2,freq\\n'\n",
        "for key, val in d.items():\n",
        "    G.add_weighted_edges_from([(key[0], key[1], val)])\n",
@@ -227,17 +148,9 @@
    },
    {
      "cell_type": "code",
-      "execution_count": 156,
+      "execution_count": null,
      "metadata": {},
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "Nodes: 576, Edges: 1257\n"
-          ]
-        }
-      ],
+      "outputs": [],
      "source": [
        "print(f\"Nodes: {G.number_of_nodes()}, Edges: {G.number_of_edges()}\")"
      ]
@@ -253,7 +166,7 @@
    },
    {
      "cell_type": "code",
-      "execution_count": 164,
+      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
@@ -263,7 +176,7 @@
    },
    {
      "cell_type": "code",
-      "execution_count": 158,
+      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
@@ -272,7 +185,7 @@
    },
    {
      "cell_type": "code",
-      "execution_count": 159,
+      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [

 %% Cell type:markdown id: tags:

 # Normclass to graph

 %% Cell type:code id: tags:

 ``` python
 import pandas as pd
 import networkx as nx
 ```

 %% Cell type:code id: tags:

 ``` python
 df = pd.read_csv('/Users/lmoncla/Nextcloud-LIRIS/GEODE/GEODE - Partage consortium/Corpus/EDdA/EDdA_dataset_articles.tsv',sep='\t')
 ```

 %% Cell type:code id: tags:

 ``` python
 len(df['edda_class'].unique())
 ```

-%% Output
-
-    2908
-
 %% Cell type:code id: tags:

 ``` python
 df = df.drop(df.loc[(df['edda_class'] == 'unclassified') | (df['edda_class'] == 'pending') | (df['edda_class'] == '0')].index)
 ```

 %% Cell type:code id: tags:

 ``` python
 len(df['edda_class'].unique())
 ```

-%% Output
-
-    2905
-
 %% Cell type:code id: tags:

 ``` python
 df = df.dropna(subset=['edda_class'])
 ```

 %% Cell type:code id: tags:

 ``` python
 len(df['edda_class'].unique())
 ```

-%% Output
-
-    2904
-
 %% Cell type:code id: tags:

 ``` python
 normclasses = df['edda_class'].unique()
 normclasses
 ```

-%% Output
-
-    array(['Grammaire', 'Ecrivains modernes', 'Calendrier Julien', ...,
-           'Jeux militaires françois', 'Gravure antique sur métal',
-           'Inscription | Médailles | Poésie'], dtype=object)
-
 %% Cell type:code id: tags:

 ``` python
 df_group = df.groupby('edda_class').size()
 ```

 %% Cell type:code id: tags:

 ``` python
 df_group
 ```

-%% Output
-
-    edda_class
-    Abus des langues                     1
-    Accord de sons                       1
-    Acoustique                           6
-    Agonistique                          1
-    Agriculture                        127
-                                      ...
-    terme usité parmi les Maréchaux      1
-    vaisselle d'étain                    1
-    Ébénisterie                          3
-    Ébénisterie | Tapisserie             1
-    Économie rustique                    3
-    Length: 2904, dtype: int64
-
 %% Cell type:code id: tags:

 ``` python
 d = {}
 for normclass, freq in df_group.items():
-    #print(normclass, freq)
+    print(normclass, freq)

    try:
        n = [x.strip() for x in normclass.split('|')] # corrige le probleme du normclass 'Géographie moderne  |'

-        for i in range(1,len(n)):
-            t = (n[0],n[i])
-            if t not in d:
-                d[t] = freq
-            else:
-                d[t] += freq
+        for i in range(0,len(n)):
+            for j in range(i+1,len(n)):
+                t = (n[i],n[j])
+                if t not in d:
+                    d[t] = freq
+                else:
+                    d[t] += freq

    except AttributeError:
        print(normclass)
 ```

 %% Cell type:code id: tags:

 ``` python
 len(d.keys())
 ```

-%% Output
-
-    1257
-
 %% Cell type:code id: tags:

 ``` python
-G = nx.DiGraph()
+G = nx.Graph()
 content = 'node1,node2,freq\n'
 for key, val in d.items():
    G.add_weighted_edges_from([(key[0], key[1], val)])
    content += key[0]+','+key[1]+','+str(val)+'\n'
 ```

 %% Cell type:code id: tags:

 ``` python
 print(f"Nodes: {G.number_of_nodes()}, Edges: {G.number_of_edges()}")
 ```

-%% Output
-
-    Nodes: 576, Edges: 1257
-
 %% Cell type:code id: tags:

 ``` python
 print(list(G.edges()))
 ```

 %% Cell type:code id: tags:

 ``` python
 with open("../data/normclass_network.csv", 'w') as f:
    f.write(content)
 ```

 %% Cell type:code id: tags:

 ``` python
 nx.write_gexf(G, "../data/normclass_network.gexf")
 ```

 %% Cell type:code id: tags:

 ``` python
 nx.write_gml(G, "../data/normclass_network.graphml")
 ```

 %% Cell type:code id: tags:

 ``` python
 ```

 %% Cell type:code id: tags:

 ``` python
 ```

 %% Cell type:code id: tags:

 ``` python
 ```

 %% Cell type:code id: tags:

 ``` python
 ```

 %% Cell type:code id: tags:

 ``` python
 ```