updated interface with prediction of ev in popup

predihood/charts.py

@@ -20,7 +20,7 @@ def generate_charts():
     data.generate_from_cities(cities)
     lists = get_selected_indicators_lists(10)
     for j, env in enumerate(["social"]):
-        dataset = Dataset(data, env, selected_indicators=lists["10"][env], train_size=0.8, test_size=0.2)
+        dataset = Dataset(data, env, selected_indicators=lists["10"][env], train_size=0.8, test_size=0.2, _type="unsupervised")
         dataset.init_all_in_one()
         algo = Chart(name='chart', dataset=dataset, number_of_iris=len(cities))
         algo.compute_trendline()

predihood/classes/Chart.py

@@ -80,45 +80,45 @@ def similarity(data1, data2, max_distance, nb_points):
 
 
 class Chart(Method):
-    def __init__(self, name, dataset, number_of_iris=12):
+    def __init__(self, name, dataset, number_of_iris=16):
         Method.__init__(self, name, dataset)
         self.chart = None
         self.dataset = dataset
         # set all NaN values as 0
         self.dataset.data = self.dataset.data.fillna(0)
-        # self.number_of_iris = number_of_iris if number_of_iris % 2 == 0 else 12
+        self.number_of_iris = number_of_iris if number_of_iris % 2 == 0 else 12
         self.iris_per_line = 2
         self.step = 0
         self.max_value = 0
 
     def compute_trendline(self):
         # for indicator in self.dataset.selected_indicators:
-        # fig, axs = plt.subplots(int(self.number_of_iris / 2), self.iris_per_line, figsize=(15, 15))  # rows, columns
+        fig, axs = plt.subplots(int(self.number_of_iris / 2), self.iris_per_line, figsize=(15, 15))  # rows, columns
         i, j, z = 0, 0, 1  # i and j are indices to plot sub-figures and z is the counter to place figures
         all_relevant_indicators = []
         n = 0
-        for index, row in self.dataset.data.iterrows():  # head(self.number_of_iris).
+        for index, row in self.dataset.data.head(self.number_of_iris).iterrows():  # head(self.number_of_iris).
             data = []
             list_indicators = self.dataset.selected_indicators if self.dataset.selected_indicators is not None else self.dataset.indicators
             for indicator in list_indicators:
                 data.append(row[indicator])
-            # max_value = self.dataset.data[self.dataset.selected_indicators].values.max()  # .head(self.number_of_iris)
-            # logging.debug("max value is %.4f", max_value)
-            # x = np.arange(0, len(data))
-            # y = data
+            max_value = self.dataset.data.head(self.number_of_iris)[self.dataset.selected_indicators].values.max()  # .head(self.number_of_iris)
+            logging.debug("max value is %.4f", max_value)
+            x = np.arange(0, len(data))
+            y = data
             mean_of_data = round(sum(data) / len(data), 3)
-            # logging.debug("y = %s", y)
-            # f = interp1d(x, y)
-            # axs[i, j].axis(ymin=0, ymax=max_value)
-            # axs[i, j].set_xticks(np.arange(0, len(data)))
-            # self.max_value = max_value
-            # self.step = max_value / 5
-            # axs[i, j].set_yticks(np.arange(0, max_value, step=self.step))
-            # axs[i, j].plot(x, data, 'o', x, f(x), '-')
-            # for k, v in enumerate(data):
-            #     # label = # "{:.1E}".format(v)
-            #     axs[i, j].annotate(round(v, 2), (k, v), )
-            # title = str(row['CODE']) + " - " + str(self.dataset.env) + " -" + str(mean_of_data)
+            logging.debug("y = %s", y)
+            f = interp1d(x, y)
+            axs[i, j].axis(ymin=0, ymax=max_value)
+            axs[i, j].set_xticks(np.arange(0, len(data)))
+            self.max_value = max_value
+            self.step = max_value / 5
+            axs[i, j].set_yticks(np.arange(0, max_value, step=self.step))
+            axs[i, j].plot(x, data, 'o', x, f(x), '-')
+            for k, v in enumerate(data):
+                # label = # "{:.1E}".format(v)
+                axs[i, j].annotate(round(v, 2), (k, v), )
+            title = str(row['CODE']) + " - " + str(self.dataset.env) + " -" + str(mean_of_data)
             relevant_indicators = []
             for ind in range(len(list_indicators)):
                 indicator = list_indicators[ind]
@@ -132,15 +132,15 @@ class Chart(Method):
                 all_relevant_indicators = union(all_relevant_indicators, relevant_indicators)
             n += 1
 
-        #     axs[i, j].set_title(title)
-        #     if z < self.iris_per_line:
-        #         z += 1
-        #         j += 1
-        #     else:
-        #         z = 1
-        #         i += 1
-        #         j = 0
-        # fig.show()
+            axs[i, j].set_title(title)
+            if z < self.iris_per_line:
+                z += 1
+                j += 1
+            else:
+                z = 1
+                i += 1
+                j = 0
+        fig.show()
         print(all_relevant_indicators)
         print(len(all_relevant_indicators))
         # self.compute_similarity()

predihood/classes/Data.py

@@ -348,13 +348,13 @@ if __name__ == '__main__':
     data = Data()
 
     cities = {
-        "tassin": ["maisons", "résidentiel", "espaces verts", "périurbain", "ouest lyon", "moyen-sup"],
-        "107 rue jean voillot villeurbanne": ["grands ensembles", "autres activités", "urbanisé", "urbain", "est lyon", "popu"],
+        # "tassin": ["maisons", "résidentiel", "espaces verts", "périurbain", "ouest lyon", "moyen-sup"],
+        # "107 rue jean voillot villeurbanne": ["grands ensembles", "autres activités", "urbanisé", "urbain", "est lyon", "popu"],
         "saint cyr au mont d'or": ["maisons", "résidentiel", "arboré", "périurbain", "ouest lyon", "sup"],
-        "doua villeurbanne": ["immeubles", "autres activités", "urabnisé", "central", "est-lyon", "moyen-inf"],
+        # "doua villeurbanne": ["immeubles", "autres activités", "urabnisé", "central", "est-lyon", "moyen-inf"],
         "part dieu lyon": ["immeubles", "commerçant", "urbanisé", "centrtal", "centre lyon", "moyen"],
-        "dompierre sur besbre": ["maisons", "résidentiel", "arboré", "rural", "est moulins", "sup"],
-        "rue de la favorite lyon": ["mixte", "commercant", "urbanisé", "urbain", "nord-ouest lyon", "moyen"],
-        "lezoux": ["maisons", "résidentiel", "arboré", "rural", "est clermont-ferrand", "moyen-sup"]
+        # "dompierre sur besbre": ["maisons", "résidentiel", "arboré", "rural", "est moulins", "sup"],
+        # "rue de la favorite lyon": ["mixte", "commercant", "urbanisé", "urbain", "nord-ouest lyon", "moyen"],
+        # "lezoux": ["maisons", "résidentiel", "arboré", "rural", "est clermont-ferrand", "moyen-sup"]
     }
     data.generate_from_cities(cities)

predihood/classes/MethodSelection.py

@@ -62,10 +62,11 @@ class MethodSelection(Method):
             upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))
 
             self.best_indicators = []
+            print(upper)
             for i in range(len(upper.columns)):
                 column = upper.columns[i]
                 for k, value in upper[column].items():
-                    if value == 1 and column not in self.best_indicators:
+                    if value == 1 and column not in self.best_indicators: # and (column, k) not in self.best_indicators and (k, column) not in self.best_indicators:
                         self.best_indicators.append(column)
 
             if TITLES: plt.title("Matrice de corrélation : filtrage = " + (

predihood/generated_files/datasets/.~lock.data_density_filtered.csv#

-,nelly,MacBook-Pro.local,29.05.2020 17:18,file:///Users/nelly/Library/Application%20Support/LibreOffice/4;
\ No newline at end of file

predihood/generated_files/selected-indicators/.~lock.selection-distribution.csv#

-,nelly,MacBook-Pro.local,29.05.2020 15:26,file:///Users/nelly/Library/Application%20Support/LibreOffice/4;
\ No newline at end of file

predihood/generated_files/selected-indicators/selection-distribution.csv

-P14_LOG,DENSITY,P14_RP_SDB,P14_POP15P,DEC_D914,P14_ACTOCC15P,C14_ACTOCC1564,DEC_MED14,P14_MAISON,DEC_D814,DEC_D614,P14_POP30P,P14_POP_FR,P14_NBPI_RPMAISON,P14_NBPI_RP_ANEM10P,DEC_D414,DEC_D114,P14_ANEM_RP_LOCHLMV,P14_PMEN_ANEM10P,C14_PMEN_MENCOUPAENF,P14_RPAPPART_ACHTOT,P14_APPART,P14_NBPI_RP,P14_RPAPPART,C14_MEN,P14_ANEM_RP_PROP,DEC_Q314,DEC_D714,P14_PMEN,P14_POP2064,P14_NPER_RP_PROP,P14_MEN,P14_POP,C14_ACTOCC15P,P14_NPER_RP,DEC_D314,P14_POPMEN15P,P14_RP,P14_NSCOL15P,C14_PMEN_MENFAM,P14_POP1564,P14_NBPI_RP_ANEM0509,AREA,C14_POP15P,P14_ACTOCC1564,P14_ANEM_RP_LOC,C14_ACT1564,C14_PMEN,P14_NBPI_RPAPPART,P14_RP_ACHTOT,P14_ANEM_RP,DEC_Q114,P14_ACT1564,DEC_D214,P14_RSECOCC,P14_NBPI_RP_ANEM0204
+P14_RP,DENSITY,C14_MEN,C14_ACTOCC1564,P14_MEN,P14_LOG,DEC_Q314,DEC_D214,P14_RP_ACHTOT,P14_NBPI_RP_ANEM0509,P14_ANEM_RP_PROP,P14_POP1564,P14_RP_SDB,DEC_D114,P14_NPER_RP_PROP,DEC_D314,C14_PMEN_MENFAM,P14_POP30P,P14_NBPI_RPMAISON,P14_NBPI_RP_ANEM0204,C14_ACT1564,AREA,P14_POP,DEC_D414,P14_NBPI_RPAPPART,P14_POP15P,P14_ACTOCC1564,P14_PMEN,P14_ANEM_RP_LOCHLMV,P14_RPAPPART_ACHTOT,P14_NSCOL15P,DEC_D714,P14_RPAPPART,DEC_D614,DEC_D814,DEC_MED14,P14_NBPI_RP_ANEM10P,P14_PMEN_ANEM10P,P14_NBPI_RP,P14_ANEM_RP,DEC_Q114,P14_ACTOCC15P,C14_PMEN_MENCOUPAENF,P14_NPER_RP,P14_POPMEN15P,P14_MAISON,P14_ACT1564,P14_ANEM_RP_LOC,C14_POP15P,P14_RSECOCC,P14_APPART,P14_POP_FR,DEC_D914,P14_POP2064,C14_ACTOCC15P,C14_PMEN

predihood/main.py

@@ -96,12 +96,12 @@ def run_algorithm():
 @app.route('/predict_iris', methods=["GET"])
 def predict_iris():
     iris_code_to_predict = request.args['iris_code']
-    # clf_name = request.args['clf']
-    # clf = get_classifier(clf_name)
+    clf_name = request.args['algorithm_name']
+    clf = get_classifier(clf_name)
 
     data = Data(normalize="density", filter=True)
     data.init_all_in_one()
-    predictions = predict_one_iris(iris_code_to_predict, data, KNeighborsClassifier(n_neighbors=30), 0.8, 0.2, False)  # clf
+    predictions = predict_one_iris(iris_code_to_predict, data, clf, 0.8, 0.2, False)  # clf
     return {"predictions": predictions}
 
 

predihood/predict.py

@@ -347,11 +347,11 @@ def expe4(data, clf, train_size, test_size, remove_outliers=False):
         results[env] = OrderedDict()
         logging.debug("--- %s ---", env)
 
-        dataset = Dataset(data_not_filtered, env, selected_indicators=data_not_filtered.indicators, train_size=train_size, test_size=test_size, outliers=remove_outliers)
+        dataset = Dataset(data_not_filtered, env, selected_indicators=data_not_filtered.indicators, train_size=train_size, test_size=test_size, outliers=remove_outliers, _type='supervised')
         dataset.init_all_in_one()
 
         mean_classifier = 0.0
-        algo = MethodPrediction(name="", dataset=dataset, classifier=clf, _type='supervised')
+        algo = MethodPrediction(name="", dataset=dataset, classifier=clf)
         algo.fit()
         algo.compute_performance()
         results[env]["accuracy_none"] = algo.accuracy
@@ -359,9 +359,9 @@ def expe4(data, clf, train_size, test_size, remove_outliers=False):
         logging.debug("accuracy for %s without filtering: %f", env, algo.accuracy)
         predictions[env] = []
         for top_k, lst in lists.items():
-            dataset = Dataset(data, env, selected_indicators=lst[env], train_size=train_size, test_size=test_size, outliers=remove_outliers)
+            dataset = Dataset(data, env, selected_indicators=lst[env], train_size=train_size, test_size=test_size, outliers=remove_outliers, _type='supervised')
             dataset.init_all_in_one()
-            algo2 = MethodPrediction(name='', dataset=dataset, classifier=clf, _type='supervised')
+            algo2 = MethodPrediction(name='', dataset=dataset, classifier=clf)
             # logging.debug("size of X_train: %d", len(algo2.dataset.X_train.columns))
             # logging.debug(algo2.dataset.X_train.columns)
             algo2.fit()
@@ -435,6 +435,7 @@ def predict_one_iris(iris_code, data, clf, train_size, test_size, remove_outlier
             algo.predict(iris_code)
             predictions_lst.append(algo.prediction)
         predictions[env] = get_most_frequent(predictions_lst)  # get the most frequent value, i.e. choose among the result of each list
+    print(predictions)
     return predictions
 
 
@@ -471,8 +472,8 @@ def predict_k_means(data, iris_code):
                 kmeans.fit()
                 kmeans.predict(iris_code)
                 print(kmeans.classifier.labels_)
-                # chart = Chart(dataset=dataset, name='')
-                # chart.compute_trendline()
+                chart = Chart(dataset=dataset, name='')
+                chart.compute_trendline()
 
 
 if __name__ == '__main__':

predihood/selection.py

@@ -74,13 +74,13 @@ def generate_lists():
     data.init_all_in_one()
 
     # # 2. Run heat map and get less correlated indicators
-    dataset = Dataset(data, "batiment")  # WARNING: fill _type parameter
+    dataset = Dataset(data, "batiment", 'unsupervised')  # WARNING: fill _type parameter
     dataset.init_all_in_one()
     heat_map = MethodSelection(name="heat map EV-agnostic", dataset=dataset, parameters=PARAMETERS)
     heat_map.results()
     # heat_map.draw_and_save()
     fully_correlated_indicators = heat_map.best_indicators
-    logging.debug("fully correlated indicators: %s", fully_correlated_indicators)
+    logging.info("fully correlated indicators: %d %s", len(fully_correlated_indicators), fully_correlated_indicators)
 
     hierarchy = pd.read_csv(FILE_HIERARCHY)
 
@@ -96,7 +96,7 @@ def generate_lists():
             # logging.debug("%s { threshold_HM: %f, min_col_HM: %f, top_k: %d}", env, PARAMETERS["threshold_HM"], PARAMETERS["min_col_HM"], PARAMETERS["top_k"])
 
             # B. FEATURE IMPORTANCE on uncorrelated indicators (the ones that are not chosen by heat map) to select the most relevant ones
-            dataset = Dataset(data, env, indicators_to_remove=fully_correlated_indicators)  # WARNING: fill _type parameter
+            dataset = Dataset(data, env, indicators_to_remove=fully_correlated_indicators, _type="supervised")  # WARNING: fill _type parameter
             dataset.init_all_in_one()
 
             # a. get best indicators for ET
@@ -117,9 +117,8 @@ def generate_lists():
             primary_FI = best_indicators_FI_ET  # [indicator[0] for indicator in best_indicators_FI_ET]
             indic_ET = [best_indicators_FI_ET[i][0] for i in range(len(best_indicators_FI_ET))]
             for i in range(len(best_indicators_FI_RF)):
-                # [['indic1', score1], ['indic2, score2], ...]
-                index_indicator_in_ET = indic_ET.index(best_indicators_FI_RF[i][0]) if best_indicators_FI_RF[i][
-                                                                                           0] in indic_ET else -1
+                # [['indic1', score1], ['indic2', score2], ...]
+                index_indicator_in_ET = indic_ET.index(best_indicators_FI_RF[i][0]) if best_indicators_FI_RF[i][0] in indic_ET else -1
                 if index_indicator_in_ET >= 0:
                     primary_FI[index_indicator_in_ET][1] += best_indicators_FI_RF[i][1]
                 else:
@@ -143,7 +142,6 @@ def generate_lists():
 def selection_by_distribution(dataset):
     # for indicator in self.dataset.selected_indicators:
     # fig, axs = plt.subplots(int(self.number_of_iris / 2), self.iris_per_line, figsize=(15, 15))  # rows, columns
-    i, j, z = 0, 0, 1  # i and j are indices to plot sub-figures and z is the counter to place figures
     all_relevant_indicators = []
     n = 0
     for index, row in dataset.data.iterrows():  # head(self.number_of_iris).
@@ -175,7 +173,9 @@ def selection_by_distribution(dataset):
 
 if __name__ == '__main__':
     # generate_all_data()
+
     # generate_lists()
+
     data = Data(normalize="density", filter=True)
     data.init_all_in_one()
     dataset = Dataset(data, "batiment", "unsupervised")

predihood/static/js/carto.js

@@ -74,35 +74,91 @@ function resetHighlightAll() {
 	}
 }
 
-function showPredictions(e) {
+
+function displayPopup(e) {
+    alert("ici")
     var layer = e.target;
     var code_iris = layer.feature.properties.CODE_IRIS
-    var algorithm = $("#selectAlgorithmTooltip option:selected").text();
-    console.log(algorithm)
-    let predictions = predict(code_iris)
-
-	let messageTooltip = '<div>CODE IRIS : ' + layer.feature.properties.CODE_IRIS + '<br/>'
-    messageTooltip += 'IRIS : ' + layer.feature.properties.NOM_IRIS + '<br/>'
-    messageTooltip += 'COMMUNE : ' + layer.feature.properties.NOM_COM + '<br/><br/>'
-    messageTooltip += '<select id="selectAlgorithmTooltip">'
-    messageTooltip += "<option value='undefined'>" + "---" + "</option>"
+    var algorithm = $("#selectAlgorithmTooltip option:selected").val();
+    console.log(algorithm);
+    let predictions = undefined;
+    if (algorithm !== "undefined" && algorithm !== undefined) {
+        predictions = predict(code_iris, algorithm)
+        console.log(predictions)
+    }
+    let divInformation = $("<div>");
+    divInformation
+        .append("CODE IRIS : " + layer.feature.properties.CODE_IRIS).append($("<br>"))
+        .append("IRIS : " + layer.feature.properties.NOM_IRIS).append($("<br>"))
+        .append("COMMUNE : " + layer.feature.properties.NOM_COM).append($("<br>"))
+        .append($("<a>")
+            .prop("href", "details-iris.html?code_iris='"+layer.feature.properties.CODE_IRIS + "'"))
+            .prop("target", "_blank")
+            .val("Plus de détails")
+        .append($("<br>"));
+
+    let selectAlgorithm = $("<select>")
+    selectAlgorithm
+        .prop("id", "selectAlgorithmTooltip")
+        .append($("<option>").prop("value", "undefined").text("---"))
+
     for(let algorithm of classifiers) {
-        console.log(algorithm)
-        messageTooltip += "<option value=" + algorithm + ">" + algorithm + "</option>"
+       selectAlgorithm.append($("<option>").prop("value", algorithm).text(algorithm));
+    }
+    if (algorithm !== "undefined" && algorithm !== undefined) {
+        selectAlgorithm.val(algorithm); // select the algorithm inside the list
     }
-    messageTooltip += "</select>"
-    for(let key in predictions) { messageTooltip += key+': ' + predictions[key] + '<br/>' }
-    messageTooltip += '<a href="details-iris.html?code_iris='+layer.feature.properties.CODE_IRIS + '" target="_blank">Plus de détails</a></div>';
+
+    let divPredictions = $("<div>").prop("id", "divPredictions")
+    if(predictions !== undefined) {
+        for(let key in predictions) { divPredictions.append(key+': ' + predictions[key]["most_frequent"] + " (" + predictions[key]["count_frequent"] + "/7)").append($('<br>')); }
+    }
+
+    // let divPredictions = updatePopup(e)
+
+    let messageTooltip = divInformation[0].outerHTML + selectAlgorithm[0].outerHTML + divPredictions[0].outerHTML;
+    console.log(messageTooltip)
     layer.bindPopup(messageTooltip)
     layer.bringToFront();
     layer.openPopup();
-}
-
-$("#selectAlgorithmTooltip").on("change", function() {
-   alert("changed")
-   eventsIRIS()
-});
 
+    $("#selectAlgorithmTooltip").on("click", function() { displayPopup(e)}) // update popup (env variables) when click on an algorithm
+
+    // $("#selectAlgorithmTooltip").change(function() {
+        // var layer = e.target;
+        // var code_iris = layer.feature.properties.CODE_IRIS
+        // var algorithm = $("#selectAlgorithmTooltip option:selected").text();
+        // console.log(algorithm)
+        // let predictions = predict(code_iris, algorithm)
+        //
+        // // let messageTooltip = '<div>CODE IRIS : ' + code_iris + '<br/>'
+        // // messageTooltip += 'IRIS : ' + layer.feature.properties.NOM_IRIS + '<br/>'
+        // // messageTooltip += 'COMMUNE : ' + layer.feature.properties.NOM_COM + '<br/>'
+        // // messageTooltip += '<a href="details-iris.html?code_iris='+layer.feature.properties.CODE_IRIS + '" target="_blank">Plus de détails</a></div></br>';
+        // // var select_algorithm = $("<select>").prop("id", "selectAlgorithmTooltip");
+        // // select_algorithm.append($("<option>").val("undefined").text("---"));
+        // // for(let algorithm of classifiers) {
+        // //    select_algorithm.append($("<option>").val(algorithm).text(algorithm))
+        // // }
+        // // messageTooltip += select_algorithm[0].outerHTML
+        // // messageTooltip += "<br/>"
+        // for(let key in predictions) { $("#divPredictions").append( key+': ' + predictions[key] + '<br/>' )}
+        // messageTooltip += $("#divPredictions")[0].outerHTML;
+        // layer._popup.setContent('something else')
+        // layer.closePopup();
+        // layer.bringToFront();
+        // layer.openPopup();
+    // })
+}
+
+function updatePopup(predictions) {
+    alert("coucou")
+    // let divPredictions = $("<div>").prop("id", "divPredictions")
+    // if(predictions !== undefined) {
+    //     for(let key in predictions) { divPredictions.append(key+': ' + predictions[key]).append($('<br>')); }
+    // }
+    // return divPredictions
+}
 
 /**
  * Add IRIS layer from GeoJSON data
@@ -136,7 +192,7 @@ function eventsIRIS(feature, layer) {
 		//mouseover: highlightFeature,
 		//mouseout: resetHighlight,
 		//click: clickProperties
-        click: showPredictions
+        click: displayPopup //showPredictions
 	});
 }
 

predihood/static/js/prediction.js

-function predict(iris_code) {
+function predict(iris_code, algorithm_name) {
 	let predictions = null
     $.ajax({
 		type: "GET",
 		url: "/predict_iris",
 		data: {
-			'iris_code': iris_code
+			'iris_code': iris_code,
+			'algorithm_name': algorithm_name
 		},
 		"async": false,
 		contentType: 'application/json;charset=UTF-8',

predihood/utility_functions.py

@@ -306,7 +306,9 @@ def get_most_frequent(lst):
     """Get the most frequent item in a list. If many elements are frequent, it returns the first one.
     :param lst: the list to find the most frequent element.
     """
-    return max(set(lst), key=lst.count)
+    most_frequent_element = max(set(lst), key=lst.count)
+    dictionary = {"most_frequent": most_frequent_element, "count_frequent": lst.count(most_frequent_element)}
+    return dictionary
 
 
 #################### plot functions ####################