accuracy

code/nary_model/BPR_model.py

@@ -137,6 +137,7 @@ class BPRModel(nn.Module):
         all_labels = None
         all_decisions = None
         first = True
+        all_preferences = None
         for k in range(num_kc):
             trips = test_triplets[k]
             y = y_test[k]
@@ -162,6 +163,11 @@ class BPRModel(nn.Module):
                     comp = negative_scores < positive_scores
                     comp = comp.cpu()
                     all_decisions = comp
+                    positive_scores = positive_scores.cpu()
+                    negative_scores = negative_scores.cpu()
+                    preferences = np.concatenate((users_batch.cpu().reshape(-1,1),items_batch.cpu().reshape(-1,1), positive_scores.detach().numpy().reshape(-1,1)), axis = 1)
+                    pref_neg = np.concatenate((users_batch.cpu().reshape(-1,1),negatives_batch.cpu().reshape(-1,1), negative_scores.detach().numpy().reshape(-1,1)), axis = 1)
+                    all_preferences = np.concatenate((preferences,pref_neg), axis = 0)
                     first = False
                 else:
                     all_labels = np.concatenate((all_labels, y_true), axis=0)
@@ -169,9 +175,14 @@ class BPRModel(nn.Module):
                     comp = negative_scores < positive_scores
                     comp = comp.cpu()
                     all_decisions = np.concatenate((all_decisions, comp), axis=0)
+                    preferences = np.concatenate((users_batch.cpu().reshape(-1,1),items_batch.cpu().reshape(-1,1), positive_scores.detach().numpy().reshape(-1,1)), axis = 1)
+                    pref_neg = np.concatenate((users_batch.cpu().reshape(-1,1),negatives_batch.cpu().reshape(-1,1), negative_scores.detach().numpy().reshape(-1,1)), axis = 1)
+                    preferences = np.concatenate((preferences,pref_neg), axis = 0)
+                    #print(preferences.shape)
+                    all_preferences = np.concatenate((all_preferences,preferences), axis=0)
                   
                 correct_ranking = sum(negative_scores < positive_scores)# for score in negative_scores)
         mse1 = mean_squared_error(all_labels, all_predictions)
         auc = roc_auc_score(all_labels, all_predictions)
-        return accuracy_score(all_labels, all_decisions), precision_score(all_labels, all_decisions),  recall_score(all_labels, all_decisions), all_decisions
+        return accuracy_score(all_labels, all_decisions), precision_score(all_labels, all_decisions),  recall_score(all_labels, all_decisions), all_decisions, all_preferences
 

code/nary_model/main_nary.py

@@ -127,41 +127,49 @@ def parse_dataframe(data, dico_kc, dico_users, dico_items, d_quest_val, nb_item_
     else:
         return dico_items, triplets, y_true
 
-def compute_pred(all_predictions, test, y_test, dico_users, dico_items):
-    r = 0
-    all_test = test[0]
-    for i in range(1,len(test)):
-        all_test = np.concatenate((test[i],all_test),axis=0)
+def compute_accuracy_multi_mod(all_preferences, dico_users, dico_items, dataTest):
+    # Remove duplicate
+    new_array = [tuple(row) for row in all_preferences]
+    all_preferences = np.unique(new_array, axis=0)
+    # Revert dictionaries
     l = list(dico_items)
+    lu = list(dico_users)
+    # Extract the questions
     list_quest = []
-    for i in range(len(all_test)):
-        q,r = parse_it(l[all_test[i,1]])
+    for i in range(len(all_preferences)):
+        q,r = parse_it(l[int(all_preferences[i,1])])
         list_quest.append(int(q))
-    all_test = np.concatenate((all_test,np.array(list_quest).reshape(-1,1)), axis=1)   
-
+    all_preferences = np.concatenate((all_preferences, np.array(list_quest).reshape(-1,1)), axis=1)   
+    # Compute the predicted value
     responses = []
-    list_user = flattern_array(all_test[:,0])
+    list_user = flattern_array(all_preferences[:,0])
     for u in list_user:
-        my_users = all_test[:,0] == u
-        list_quest = flattern_array(all_test[my_users,4])
+        my_users = all_preferences[:,0] == u
+        list_quest = flattern_array(all_preferences[my_users,3])
         for quest in list_quest: 
-            rows = all_test[:,4] == quest
+            rows = all_preferences[:,3] == quest
             my_rows = np.logical_and(rows,my_users)
-            
-            # we go the rows with all the modalities
-            modality = -1
-            for t in range(len(all_test[my_rows])):
-                ione = all_test[my_rows][t][1]
-                itwo = all_test[my_rows][t][2]
-                q1,r1 = parse_it(l[ione])
-                q2,r2 = parse_it(l[itwo])
-                if(all_predictions[my_rows][t] == 1):
-                    modality = r1
-                else:
-                    modality = r2
-            responses.append([dico_users[u], quest, modality])   
-       
-    return responses        
+            # we got the rows corresponding to a user and a question
+            # and take the row with the maximum predicted values
+            m = np.argmax(all_preferences[my_rows,2])
+            # Get the modality
+            item = int(all_preferences[my_rows][m][1])
+            q1,modality = parse_it(l[item])
+            responses.append([lu[int(u)], int(quest), int(modality)])   
+    # Sort responses
+    dfPred = pd.DataFrame(responses, columns=['user_id', 'question','modality'])
+    dfPred = dfPred.sort_values(by=['user_id', 'question'])
+    pred = dfPred['modality'].values
+    # True data
+    dfTrue = pd.read_csv(dataTest, names=['user_id', 'item_id','correct','knowledge',"question"])
+    for row_index, row in dfTrue.iterrows():
+        col = row['item_id']
+        q,r = parse_it(col)
+        dfTrue.at[row_index,'question'] = int(q)
+    dfTrue = dfTrue.drop_duplicates(subset=['user_id', 'question'])  
+    dfTrue = dfTrue.sort_values(by=['user_id', 'question'])
+    print("Accuracy multi-modal", accuracy_score(dfTrue['correct'].values, pred), "RMSE", np.sqrt(mean_squared_error(dfTrue['correct'].values, pred)))
+               
 
         
    
@@ -201,19 +209,12 @@ if __name__ == '__main__':
     new_embedding_value = bpr_model.user_embeddings.weight.clone().detach().cpu().numpy()
     write_file_doa(filename, new_embedding_value, train, dico_kc, dico_users, dico_items)  
     doa, rdoa = compute_doa(filename)
-    # write embed items
-    #new_embedding_items = bpr_model.item_embeddings.weight.clone().detach().cpu().numpy()
-    #write_file(filename+"embedding_items.csv", new_embedding_items[0:nb_item_train])
-    
-    #write_file_std(FileName1+"kc_emb.txt", dico_items)
-    #print("Accuracy and Doa on train dataset:", acc, doa)
+      
     # Test
-    acc, precision, rappel, all_predictions = bpr_model.evaluate_model(test, len(dico_kc), y_test) 
+    acc, precision, rappel, all_predictions, all_preferences = bpr_model.evaluate_model(test, len(dico_kc), y_test) 
     s = str(acc) +","+ str( precision)+ ","+str(rappel)+ ","+str(doa)
-    #for i in range(len(rdoa)):
-    #    s = s + ','+ str(rdoa[i])
     print(s)
-
+    compute_accuracy_multi_mod(all_preferences, dico_users, dico_items, dataTest)
     # coding users and kc from user_label.csv
     '''
     os.getcwd()
@@ -232,6 +233,6 @@ if __name__ == '__main__':
             questEval[ind, kc] = val  
     write_file(filename+"_user_quest_label.csv", questEval)
     '''
-the_predictions = compute_pred(all_predictions, test, y_test, dico_users, dico_items)
-file = "_test_predictions_"+str(alpha)+".csv"
-write_file(filename + file, np.array(the_predictions))
\ No newline at end of file
+#the_predictions = compute_pred(all_predictions, test, y_test, dico_users, dico_items)
+#file = "_test_predictions_"+str(alpha)+".csv"
+#write_file(filename + file, np.array(the_predictions))
\ No newline at end of file