update code + ADD dropdown menu in geocoding app

geocoder_app.py

-from flask import Flask, escape, request, render_template,jsonify,Markup, redirect
-from lib.geocoder import Geocoder,TextGeocoder,heuristic_mean
+from flask import Flask, escape, request, render_template,jsonify,Markup, redirect, url_for
+from lib.geocoder.our_geocoder import Geocoder,TextGeocoder
+from lib.geocoder.heuristics import *
 
+import spacy
 
 app = Flask(__name__)
 
@@ -14,31 +16,36 @@ dict_model = {
     "GB_C":("./outputs/GB_MODEL_2/GB.txt_100_4_100__C.h5","./outputs/GB_MODEL_2/GB.txt_100_4_100__C_index"),
     "GB_AC":("./outputs/GB_MODEL_2/GB.txt_100_4_100__A_C.h5","./outputs/GB_MODEL_2/GB.txt_100_4_100__A_C_index"),
     "GB_IC":("./outputs/GB_MODEL_2/GB.txt_100_4_100__I_C.h5","./outputs/GB_MODEL_2/GB.txt_100_4_100__I_C_index")
-    ,"FR_IGN":("./outputs/IGN_4_100_A_C.h5","./outputs/IGN_4_100_A_C_index")
+    ,"FR_IGN":("./outputs/IGN/onlyAdjac/IGN_4_100_A_C.h5","./outputs/IGN/onlyAdjac/IGN_4_100_A_C_index")
 }
 
-MODEL = "FR_IGN"
+MODEL = "FR_AC"
 LANG = "fr"
 NER = "spacy"
 
+heuristic_func = heuristic_cluster
 
 geocoder = Geocoder(*dict_model[MODEL])
-g_t = TextGeocoder(geocoder,NER,LANG,heuristic_mean)
+g_t = TextGeocoder(geocoder,NER,LANG,heuristic_func)
+
 
 @app.route('/')
 def home():
     toponym = request.args.get("top", "")
     c_toponym = request.args.get("c_top", "")
+    msg = request.args.get("msg", "")
+    msg_code = request.args.get("msg_code", "info")
     if toponym and c_toponym:
-        lon,lat = geocoder.get_coord(toponym,c_toponym)
-        lon,lat = geocoder.wgs_coord(lon,lat)
-        return  render_template("pair_topo.html",lat=lat,lon=lon,title="Toponyms Pair Geocoder")
+        lon,lat = geocoder.get_coords([toponym],[c_toponym])
+        lon,lat = lon[0],lat[0]
+        print(lon,lat)
+        return  render_template("pair_topo.html",lat=lat,lon=lon,title="Toponyms Pair Geocoder",dict_model=dict_model,msg_code=msg_code)
     else:
-        return  render_template("pair_topo.html",title="Toponyms Pair Geocoder")
+        return  render_template("pair_topo.html",title="Toponyms Pair Geocoder",dict_model=dict_model,msg_code=msg_code)
 
 @app.route('/text')
 def text():
-    return render_template("text.html",title="Text Geocoder")
+    return render_template("text.html",title="Text Geocoder",dict_model=dict_model)
 
 @app.route('/geocode', methods=['POST', 'GET'])
 def geocode():
@@ -58,8 +65,29 @@ def geocode():
             if r["text"] in place_coords:
                 continue
             place_coords[r["text"]]={"lat":float(r["coord"]["lat"]),"lon":float(r["coord"]["lon"])}
-        return render_template("text.html",title="Text Geocoder",data={"type":"success","output":Markup(html_),"place_coords":place_coords})
+        return render_template("text.html",title="Text Geocoder",data={"type":"success","output":Markup(html_),"place_coords":place_coords},dict_model=dict_model)
+
 
+@app.route("/loadmodel/<model_id>")
+def loadModel(model_id):
+    global geocoder,g_t,LANG
+    if not model_id in dict_model:
+        return redirect(url_for(".home",msg="An error happend when loading the model \"{0}\"!".format(model_id),msg_code="danger"))
+    else: 
+        geocoder = Geocoder(*dict_model[model_id])
+        g_t = TextGeocoder(geocoder,NER,LANG,heuristic_func)
+        return redirect(url_for(".home",msg="Model \"{0}\" was loaded successfuly!".format(model_id),msg_code="success"))
+
+@app.route("/loadlang/<lang>")
+def loadLang(lang):
+    global geocoder,g_t,LANG
+    try:
+        g_t = TextGeocoder(geocoder,NER,lang,heuristic_func)
+        LANG = lang
+        return redirect(url_for(".home",msg="Language is now set to \"{0}\"!".format(LANG),msg_code="success"))
+    except:
+        return redirect(url_for(".home",msg="\"{}\" language is not available!".format(lang),msg_code="danger"))
+        
 
 if __name__ == "__main__":
-    app.run(host="0.0.0.0")
\ No newline at end of file
+    app.run(host="0.0.0.0",debug=True)
\ No newline at end of file

lib/geocoder/heuristics.py

+import pandas as pd
+import numpy as np
+
+from haversine import haversine_vector, Unit
+from sklearn.cluster import DBSCAN
+
+def heuristic_mean(geocoder,toponyms):
+    input_ = np.asarray([[t1,t2] for t2 in toponyms for t1 in toponyms if t2 != t1])
+    res_geocode = pd.DataFrame(input_,columns="t tc".split())
+    lons,lats = geocoder.get_coords(input_[:,0],input_[:,1])
+    res_geocode["lon"] = lons
+    res_geocode["lat"] = lats
+    results = {}
+    for tp in toponyms:
+        lat = res_geocode[res_geocode.t == tp].lat.mean()
+        lon = res_geocode[res_geocode.t == tp].lon.mean()
+        results[tp]={"lat":lat,"lon":lon}
+    return results
+
+def heuristic_no_context(geocoder,toponyms):
+    input_ = np.asarray([[t1,t1] for t2 in toponyms for t1 in toponyms if t2 != t1])
+    res_geocode = pd.DataFrame(input_,columns="t tc".split())
+    lons,lats = geocoder.get_coords(input_[:,0],input_[:,1])
+    res_geocode["lon"] = lons
+    res_geocode["lat"] = lats
+    results = {}
+    for tp in toponyms:
+        lat = res_geocode[res_geocode.t == tp].lat.mean()
+        lon = res_geocode[res_geocode.t == tp].lon.mean()
+        results[tp]={"lat":lat,"lon":lon}
+    return results
+
+def heuristic_cluster(geocoder,toponyms,eps=100):
+    results = {}
+    input_ = np.asarray([[t1,t2] for t2 in toponyms for t1 in toponyms if t2 != t1])
+    res_geocode = pd.DataFrame(input_,columns="t tc".split())
+    lons,lats = geocoder.get_coords(input_[:,0],input_[:,1])
+    res_geocode["lon"] = lons
+    res_geocode["lat"] = lats
+
+    clf = DBSCAN(eps=eps)
+    for t in toponyms:
+        tp_df = res_geocode[res_geocode.tc == t].copy()
+
+        coords = tp_df["lon lat".split()].values
+        clf.fit(haversine_vector(coords,coords,unit="km",comb=True))
+
+        tp_df["cluster"] = clf.labels_
+        counts_ = dict(tp_df.cluster.value_counts())
+        max_cluster = max(counts_, key=counts_.get)
+        tp_df = tp_df[tp_df.cluster == max_cluster]
+        lat = tp_df.lat.median()
+        lon = tp_df.lon.median() #
+        results[t]={"lat":lat,"lon":lon}
+    return results
\ No newline at end of file

lib/geocoder/our_geocoder.py

@@ -17,6 +17,7 @@ from lib.word_index import WordIndex
 from lib.ngram_index import NgramIndex
 from lib.utils_geo import haversine_tf_1circle
 
+
 import stanza
 import spacy
 import os
@@ -80,19 +81,7 @@ class Geocoder(object):
             ax.plot(lon,lat,marker='o', color='red', markersize=5)
             plt.show()
 
-def heuristic_mean(geocoder,data):
-    toponyms = data.text.unique()
-    input_ = np.asarray([[t1,t2] for t2 in toponyms for t1 in toponyms if t2 != t1])
-    res_geocode = pd.DataFrame(input_,columns="t tc".split())
-    lons,lats = geocoder.wgs_coord(*geocoder.get_coords(input_[:,0],input_[:,1]))
-    res_geocode["lon"] = lons
-    res_geocode["lat"] = lats
-    results = {}
-    for tp in toponyms:
-        lat = res_geocode[res_geocode.t == tp].lat.mean()
-        lon = res_geocode[res_geocode.t == tp].lon.mean()
-        results[tp]={"lat":lat,"lon":lon}
-    return results
+
     
 class TextGeocoder():
     def __init__(self,geocoder_model,ner_name,lang,heuristic_func,n_jobs=None):
@@ -116,7 +105,7 @@ class TextGeocoder():
 
     def geocode(self,entities):
         df = pd.DataFrame(entities)
-        heuristic_results = self.heuristic_func(self.geocoder_model,df)
+        heuristic_results = self.heuristic_func(self.geocoder_model,df.text.values)
         for e in range(len(entities)):
             entities[e]["coord"] = heuristic_results[entities[e]["text"]]
         return entities

templates/skeleton.html

@@ -64,11 +64,45 @@
                 <div class="navbar-nav">
                     <a class="nav-link" href="/">Toponyms Pair Geocoder</a>
                     <a class="nav-link" href="/text">Text Geocoder</a>
+
+
+                </div>
+                <div class="navbar-nav ml-auto">
+                    <li class="nav-item dropdown">
+                        <a class="nav-link dropdown-toggle" href="#" id="navbarModelDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+                          Choose Model
+                        </a>
+
+                        <div class="dropdown-menu" aria-labelledby="navbarModelDropdown">
+                            {% for id_ in dict_model %}
+                            <a class="dropdown-item" href="/loadmodel/{{id_}}">{{id_}}</a>
+                            <br>{% endfor %}
+                        </div>
+                    </li>
+                    <li class="nav-item dropdown">
+                        <a class="nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+                        Choose Lang for Spacy
+                        </a>
+                        <div class="dropdown-menu" aria-labelledby="navbarDropdown">
+                            <a class="dropdown-item" href="/loadlang/fr">fr</a>
+                            <a class="dropdown-item" href="/loadlang/en">en</a>
+                        </div>
+                    </li>
                 </div>
             </div>
         </nav>
+        {% if request.args.get("msg","") != "" %}
+        <div class="alert alert-{{msg_code}} alert-dismissible fade show" role="alert">
+            {{request.args.get("msg") }}
+            <button type="button" class="close" data-dismiss="alert" aria-label="Close">
+              <span aria-hidden="true">&times;</span>
+            </button>
+        </div>
+
+        {% endif %}
         <h2 class="text-center" style="margin-top: 0.5em;">{{title}}</h2>
-        {% block content %}{% endblock %}
+
+        <br>{% block content %}{% endblock %}
     </main>
 
     <!-- JS SCRIPTS -->

train_geocoder_v2.py

@@ -68,7 +68,7 @@ if args.adjacency:
 if args.inclusion:
     PREFIX_OUTPUT_FN += "_I"
 if args.wikipedia:
-    PREFIX_OUTPUT_FN += "_C"
+    PREFIX_OUTPUT_FN += "_P"
 
 MODEL_OUTPUT_FN = "outputs/{0}.h5".format(PREFIX_OUTPUT_FN)
 INDEX_FN = "outputs/{0}_index".format(PREFIX_OUTPUT_FN)