diff --git a/biblio.bib b/biblio.bib
index 27480653e4362cf98fbcfa1d200688f2c158d578..ce335322e4f34336b3a97cf38e208df335f70815 100644
--- a/biblio.bib
+++ b/biblio.bib
@@ -37,8 +37,10 @@ keywords = "Home buyer, Real estate website, Housing search behavior, Case-based
}
@misc{datafrance,
- author={DataFrance},
- title={}
+ title={DataFrance},
+ howpublished={https://datafrance.info/},
+ note = {https://datafrance.info/},
+ year=2018
}
@inproceedings{airbnb2017,
diff --git a/html/api.html b/html/api.html
new file mode 100644
index 0000000000000000000000000000000000000000..864ca6490754076157400a5b1a03cb3e7cdf022e
--- /dev/null
+++ b/html/api.html
@@ -0,0 +1,1646 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1" />
+<meta name="generator" content="pdoc 0.6.3" />
+<title>api API documentation</title>
+<meta name="description" content="" />
+<link href='https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.0/normalize.min.css' rel='stylesheet'>
+<link href='https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/8.0.0/sanitize.min.css' rel='stylesheet'>
+<link href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css" rel="stylesheet">
+<style>.flex{display:flex !important}body{line-height:1.5em}#content{padding:20px}#sidebar{padding:30px;overflow:hidden}.http-server-breadcrumbs{font-size:130%;margin:0 0 15px 0}#footer{font-size:.75em;padding:5px 30px;border-top:1px solid #ddd;text-align:right}#footer p{margin:0 0 0 1em;display:inline-block}#footer p:last-child{margin-right:30px}h1,h2,h3,h4,h5{font-weight:300}h1{font-size:2.5em;line-height:1.1em}h2{font-size:1.75em;margin:1em 0 .50em 0}h3{font-size:1.4em;margin:25px 0 10px 0}h4{margin:0;font-size:105%}a{color:#058;text-decoration:none;transition:color .3s ease-in-out}a:hover{color:#e82}.title code{font-weight:bold}h2[id^="header-"]{margin-top:2em}.ident{color:#900}pre code{background:#f8f8f8;font-size:.8em;line-height:1.4em}code{background:#f2f2f1;padding:1px 4px;overflow-wrap:break-word}h1 code{background:transparent}pre{background:#f8f8f8;border:0;border-top:1px solid #ccc;border-bottom:1px solid #ccc;margin:1em 0;padding:1ex}#http-server-module-list{display:flex;flex-flow:column}#http-server-module-list div{display:flex}#http-server-module-list dt{min-width:10%}#http-server-module-list p{margin-top:0}.toc ul,#index{list-style-type:none;margin:0;padding:0}#index code{background:transparent}#index h3{border-bottom:1px solid #ddd}#index ul{padding:0}#index h4{font-weight:bold}#index h4 + ul{margin-bottom:.6em}@media (min-width:200ex){#index .two-column{column-count:2}}@media (min-width:300ex){#index .two-column{column-count:3}}dl{margin-bottom:2em}dl dl:last-child{margin-bottom:4em}dd{margin:0 0 1em 3em}#header-classes + dl > dd{margin-bottom:3em}dd dd{margin-left:2em}dd p{margin:10px 0}.name{background:#eee;font-weight:bold;font-size:.85em;padding:5px 10px;display:inline-block;min-width:40%}.name:hover{background:#e0e0e0}.name > span:first-child{white-space:nowrap}.name.class > span:nth-child(2){margin-left:.4em}.inherited{color:#999;border-left:5px solid #eee;padding-left:1em}.inheritance em{font-style:normal;font-weight:bold}.desc h2{font-weight:400;font-size:1.25em}.desc h3{font-size:1em}.desc dt code{background:inherit}.source summary{color:#666;text-align:right;font-weight:400;font-size:.8em;text-transform:uppercase;cursor:pointer}.source pre{max-height:500px;overflow:auto;margin:0}.source pre code{font-size:12px;overflow:visible}.hlist{list-style:none}.hlist li{display:inline}.hlist li:after{content:',\2002'}.hlist li:last-child:after{content:none}.hlist .hlist{display:inline;padding-left:1em}img{max-width:100%}.admonition{padding:.1em .5em;margin-bottom:1em}.admonition-title{font-weight:bold}.admonition.note,.admonition.info,.admonition.important{background:#aef}.admonition.todo,.admonition.versionadded,.admonition.tip,.admonition.hint{background:#dfd}.admonition.warning,.admonition.versionchanged,.admonition.deprecated{background:#fd4}.admonition.error,.admonition.danger,.admonition.caution{background:lightpink}</style>
+<style media="screen and (min-width: 700px)">@media screen and (min-width:700px){#sidebar{width:30%}#content{width:70%;max-width:100ch;padding:3em 4em;border-left:1px solid #ddd}pre code{font-size:1em}.item .name{font-size:1em}main{display:flex;flex-direction:row-reverse;justify-content:flex-end}.toc ul ul,#index ul{padding-left:1.5em}.toc > ul > li{margin-top:.5em}}</style>
+<style media="print">@media print{#sidebar h1{page-break-before:always}.source{display:none}}@media print{*{background:transparent !important;color:#000 !important;box-shadow:none !important;text-shadow:none !important}a[href]:after{content:" (" attr(href) ")";font-size:90%}a[href][title]:after{content:none}abbr[title]:after{content:" (" attr(title) ")"}.ir a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}@page{margin:0.5cm}p,h2,h3{orphans:3;widows:3}h1,h2,h3,h4,h5,h6{page-break-after:avoid}}</style>
+</head>
+<body>
+<main>
+<article id="content">
+<header>
+<h1 class="title">Module <code>api</code></h1>
+</header>
+<section id="section-intro">
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">#!/usr/bin/env python
+# encoding: utf-8
+# =============================================================================
+# Abstraction layer for the MongoDB database
+# Performs operations such as find, update, convert_geojson_files, intersect, etc.
+# Some methods are not static because they require a valid DB connection (performed in __init__)
+# =============================================================================
+
+import pymongo
+from bson import json_util # used to convert BSON to JSON (especially ObjectId type of "_id")
+import json
+import logging
+from mongiris import config
+
+
+class Mongiris:
+ """
+ The Mongiris class is an API to manipulate data from the MongoDB `dbinsee` datatase.
+
+ Several methods accepts a 'collection' parameter for flexibility (i.e., be able to query other collections than the
+ iris collection).
+ Most methods convert resulting documents from the BSON format (MongoDB) to JSON. This mainly avoids the issue of
+ ObjectId type (for MongoDB '_id' field).
+
+ The constructor initializes the logger and it automatically connects to the database.
+ The name of the database and of the three collections
+ are based on the default names (the ones from the dump). If the names are changed in MongoDB, they should be changed
+ in the `config.py` file.
+
+ Examples of usages, including testing geospatial queries, are available in `tests/api_tests.py`.
+
+ An example of IRIS following the GeoJSON format is provided in `data/example-iris.json`.
+
+ Additional resources:
+
+ - [MongoDB documentation](http://www.mongodb.org/)
+
+ - [GeoJSON format specifications](https://geojson.org/)
+
+ - [pymongo API](http://api.mongodb.com/python/current/) between Python and MongoDB
+
+ """
+
+ def __init__(self):
+ logging.basicConfig(format='[%(levelname)s] - %(name)s - %(asctime)s : %(message)s')
+ self.logger = logging.getLogger()
+ self.logger.setLevel(logging.INFO)
+ self.connection, self.connection_status = self.init_connection() # default connection on 'localhost', 27017
+ self.database = self.connection[config.database_iris] # database for HiL project
+ self.collection_iris = self.database[config.collection_iris]
+ self.collection_indic = self.database[config.collection_indic]
+ self.collection_sources = self.database[config.collection_sources]
+
+ @staticmethod
+ def bson_to_json(doc_bson):
+ """
+ Converts the bson data to valid JSON (including the ObjectId type converted to {"$oid": <string>}).
+
+ Args:
+ doc_bson: the BSON data to be converted
+
+ Returns:
+ doc_json: a JSON object
+ """
+ doc_json = json.loads(json_util.dumps(doc_bson, json_options=json_util.RELAXED_JSON_OPTIONS))
+ return doc_json
+
+ def init_connection(self):
+ """
+ Tries to connect to MongoDB. The output connection object do not provide reliable connection status, only the
+ boolean connection_status indicates whether the connection is working or not.
+
+ Returns:
+ connection: an object with information about the connection
+ connection_status: a boolean indicating whether the connection is a success or a failure
+ """
+ connection_status = True # by default, connection is ok
+ connection = None
+ try:
+ connection = pymongo.MongoClient(serverSelectionTimeoutMS=config.max_timeout) # default localhost:27017
+ connection.server_info() # forces a query to MongoDB (for checking the connection)
+ except pymongo.errors.ServerSelectionTimeoutError as e:
+ self.logger.error('Could not connect to the MongoDB database ! Have you launched MongoDB ? ' + str(e))
+ connection_status = False
+ return connection, connection_status
+
+ @staticmethod
+ def _parse_json_to_dict(json_file_path):
+ """ Converts a JSON file denoted by json_file_path into a Python dictionary. """
+ with open(json_file_path) as data_file:
+ data = json.load(data_file)
+ data_file.close()
+ return data
+
+ @staticmethod
+ def _save_dict_to_json(json_file_path, dict_geo):
+ """ Converts and saves a Python dictionary into a JSON file denoted by json_file_path. """
+ with open(json_file_path, 'w') as data_file:
+ json.dump(dict_geo, data_file)
+
+ def create_index(self, iris_collection):
+ """ Rebuilds a geospatial index on the iris collection. Only used in case of restoration/import. """
+ self.logger.info("Creating index on 'geometry' using " + pymongo.GEOSPHERE)
+ iris_collection.create_index([("geometry", pymongo.GEOSPHERE)])
+ self.logger.info("Index created")
+
+ def count_documents(self, collection, json_query):
+ """
+ Counts the number of documents that satisfy json_query in the given collection.
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+
+ Returns:
+ count: an integer representing the number of documents
+ """
+ return collection.count_documents(json_query)
+
+ def find_one_document(self, collection, json_query):
+ """
+ Finds the first document in the given collection that satisfies json_query.
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+
+ Returns:
+ doc_json: a dictionary representing an iris, or None
+ """
+ doc = collection.find_one(json_query)
+ doc_json = Mongiris.bson_to_json(doc)
+ return doc_json
+
+ def get_iris_from_code(self, code_iris):
+ """
+ Returns the iris identified by the given code_iris.
+
+ Args:
+ code_iris: a string containing the code of the searched iris
+
+ Returns:
+ iris: a dictionary representing an iris, or None
+ """
+ iris = self.find_one_document(self.collection_iris, {"properties.CODE_IRIS": code_iris})
+ return iris
+
+ def find_documents(self, collection, json_query, json_projection=None):
+ """
+ Finds the first document in the given collection that satisfies json_query.
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ cursor = collection.find(json_query, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json
+
+ def get_random_document(self, collection):
+ """ Returns a random document from the given collection. """
+ random_iris = collection.aggregate([{"$sample": {"size": 1}}]).next()
+ doc_json = Mongiris.bson_to_json(random_iris)
+ return doc_json
+
+ def update_one_document(self, collection, json_query, json_updates):
+ """
+ Updates the first document satisfying json_query by setting new values from json_updates.
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+ json_updates: a json document containing values to be updates (using $set operator)
+
+ Returns:
+ json_result: an UpdateResult json document containing information about the operation
+ """
+ json_result = collection.update_one(json_query, json_updates)
+ return json_result
+
+ def replace_one_document(self, collection, json_query, json_replace_doc, upsert=False):
+ """
+ Replaces the first document satisfying json_query by the document json_replace_doc (their _id are identical).
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+ json_replace_doc: the replacement doc (if _id set, should be the same _id as the doc matching json_query)
+ upsert: a boolean, whether the json_replace_doc should be inserted if no document match json_query
+
+ Returns:
+ json_result: an UpdateResult json document containing information about the operation
+ """
+ json_result = collection.replace_one(json_query, json_replace_doc, upsert)
+ return json_result
+
+ def insert_one_document(self, collection, doc):
+ """
+ Inserts a new document in the collection.
+
+ Args:
+ collection: a string representing the collection name
+ doc: a dict representing the document to be added
+
+ Returns:
+ json_result: an InsertOneResult json document containing information about the operation
+ """
+ json_result = collection.insert_one(doc)
+ return json_result # eg, the new _id is in json_result.inserted_id
+
+ def delete_all(self, collection):
+ """
+ Deletes all documents in the collection.
+
+ Args:
+ collection: a string representing the collection name
+
+ Returns:
+ json_result: an DeleteResult json document containing information about the operation
+ """
+ json_result = collection.delete_many({}) # empty collection
+ return json_result
+
+ def geo_within(self, collection, geometry, json_projection=None):
+ """
+ Finds all documents from given collection and which contain totally the given geometry.
+ Cannot be used to find the IRIS containing a point (geometry must be a polygon).
+
+ Args:
+ collection: a string representing the collection name
+ geometry: a geojson geometry ("Polygon", "$box" or "MultiPolygon", NO "Point")
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ cursor = self.find_documents(collection, {"geometry": {"$geoWithin": {"$geometry": geometry}}}, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json
+
+ def geo_within_sphere(self, collection, sphere, json_projection=None):
+ """
+ Finds all documents from given collection and which contain totally the given sphere.
+ Cannot be used to find the IRIS containing a point (geometry must be a polygon, with min. 3 points).
+
+ Args:
+ collection: a string representing the collection name
+ sphere: a geojson geometry defined by a center and a radius in radians
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ cursor = self.find_documents(collection, {"geometry": {"$geoWithin": sphere}}, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json
+
+ def intersect(self, collection, geometry, json_projection=None):
+ """
+ Finds all documents from given collection and which intersect the given geometry.
+
+ Args:
+ collection: a string representing the collection name
+ geometry: a geojson geometry
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ cursor = self.find_documents(collection, {"geometry": {"$geoIntersects": {
+ "$geometry": geometry}}}, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json
+
+ @staticmethod
+ def get_geojson_point(coordinates):
+ """
+ Builds a dictionary with GeoJSON syntax for a point using the given coordinates.
+
+ Args:
+ coordinates: the coordinates (long, lat) as a list, e.g. [4.8, 45.7]
+
+ Returns:
+ point: a dictionary with GeoJSON syntax for a Point
+ """
+ return {"type": "Point", "coordinates": coordinates}
+
+ @staticmethod
+ def convert_geojson_box_to_polygon(lng1, lat1, lng2, lat2):
+ """
+ Builds a dictionary with GeoJSON syntax for a polygon using two coordinates (points south-west and north-east).
+ This method builds the polygon by adding 2 missing points (north-west and south-east) and adds the starting
+ point (south-west) to end the loop.
+ The MongoDB $box operator is not supported with 2d-spherical indexes.
+
+ Args:
+ lng1: longitude of the first point (south-west) of the box
+ lat1: latitude of the first point (south-west) of the box
+ lng2: longitude of the second point (north-east) of the box
+ lat2: latitude of the second point (north-east) of the box
+
+ Returns:
+ box: a dictionary with GeoJSON syntax for a Box
+ """
+ coordinates = [[[lng1, lat1], [lng1, lat2], [lng2, lat2], [lng2, lat1], [lng1, lat1]]]
+ return Mongiris.get_geojson_polygon(coordinates)
+
+ @staticmethod
+ def get_geojson_polygon(coordinates):
+ """
+ Builds a dictionary with GeoJSON syntax for a polygon using the given coordinates.
+ Careful: polygons must be closed ! (first coordinate must be identical to last coordinate)
+
+ Args:
+ coordinates: the coordinates (long, lat) as a list of list, e.g. [[[4.8, 45.7], [4.9, 47.8]]]
+
+ Returns:
+ polygon: a dictionary with GeoJSON syntax for a Polygon
+ """
+ return {"type": "Polygon", "coordinates": coordinates}
+
+ def point_in_which_iris(self, coordinates, json_projection=None):
+ """
+ Finds the document (IRIS) containing the given coordinates. Uses near() since geo_within() requires a Polygon.
+ Careful: the near() operator may return several iris (low probability since distance = 1 meter) and only the
+ first one is returned.
+
+ Args:
+ coordinates: an array of coordinates (long, lat)
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a json document or None
+ """
+ results = self.near(self.collection_iris, coordinates, json_projection, 1) # distance = 1 meter
+ if len(results) == 0:
+ return None
+ return results[0]
+
+ def near(self, collection, coordinates, json_projection=None, distance_max=2000):
+ """
+ Finds all documents from given collection and which are near the given geometry (according to distance_max).
+
+ Args:
+ collection: a string representing the collection name
+ coordinates: an array of coordinates (long, lat) - near only accepts Point
+ json_projection: a json document indicating the fields that appear in the results
+ distance_max: the maximum distance of resulting iris, in meters
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ geometry = Mongiris.get_geojson_point(coordinates)
+ cursor = self.find_documents(collection, {"geometry": {"$near": {
+ "$geometry": geometry, "$maxDistance": distance_max}}}, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json
+
+ @staticmethod
+ def adjacent(collection, geometry, json_projection=None, distance=20, exclude_geometry_iris=None):
+ """
+ Finds all adjacent neighbors of an iris represented by geometry.
+ No adjacent function, so use all coordinates of an iris and find the closest iris (according to distance).
+ Could be done directly with near(), but near() is less accurate and thus incomplete.
+
+ Args:
+ collection: a string representing the collection name
+ geometry: a geojson geometry (Point, Polygon, etc.)
+ json_projection: a json document indicating the fields that appear in the results
+ distance: the maximum distance for an adjacent neighbour, in meters (10 to 50 meters are fine)
+ exclude_geometry_iris: the document _id of the iris represented by geometry, if it needs to be excluded
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ results = list()
+ results_ids = list()
+ if exclude_geometry_iris is not None: # to exclude the iris represented by geometry, add it to the results ids
+ results_ids.append(exclude_geometry_iris)
+ for coords in geometry["coordinates"][0]:
+ geometry_coords = Mongiris.get_geojson_point(coords)
+ cursor = collection.find({"geometry": {"$near": {"$geometry": geometry_coords,
+ "$maxDistance": distance}}}, json_projection)
+ for doc in cursor:
+ doc_id = doc["_id"]
+ if doc_id not in results_ids: # add the new adjacent iris if not already in the results
+ results.append(doc)
+ results_ids.append(doc_id)
+ doc_json = Mongiris.bson_to_json(results)
+ return doc_json
+
+
+if __name__ == "__main__":
+ print("Run unit tests for testing the Mongiris class.")</code></pre>
+</details>
+</section>
+<section>
+</section>
+<section>
+</section>
+<section>
+</section>
+<section>
+<h2 class="section-title" id="header-classes">Classes</h2>
+<dl>
+<dt id="api.Mongiris"><code class="flex name class">
+<span>class <span class="ident">Mongiris</span></span>
+</code></dt>
+<dd>
+<section class="desc"><p>The Mongiris class is an API to manipulate data from the MongoDB <code>dbinsee</code> datatase.</p>
+<p>Several methods accepts a 'collection' parameter for flexibility (i.e., be able to query other collections than the
+iris collection).
+Most methods convert resulting documents from the BSON format (MongoDB) to JSON. This mainly avoids the issue of
+ObjectId type (for MongoDB '_id' field).</p>
+<p>The constructor initializes the logger and it automatically connects to the database.
+The name of the database and of the three collections
+are based on the default names (the ones from the dump). If the names are changed in MongoDB, they should be changed
+in the <code>config.py</code> file.</p>
+<p>Examples of usages, including testing geospatial queries, are available in <code>tests/api_tests.py</code>.</p>
+<p>An example of IRIS following the GeoJSON format is provided in <code>data/example-iris.json</code>.</p>
+<p>Additional resources:</p>
+<ul>
+<li>
+<p><a href="http://www.mongodb.org/">MongoDB documentation</a></p>
+</li>
+<li>
+<p><a href="https://geojson.org/">GeoJSON format specifications</a></p>
+</li>
+<li>
+<p><a href="http://api.mongodb.com/python/current/">pymongo API</a> between Python and MongoDB</p>
+</li>
+</ul></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">class Mongiris:
+ """
+ The Mongiris class is an API to manipulate data from the MongoDB `dbinsee` datatase.
+
+ Several methods accepts a 'collection' parameter for flexibility (i.e., be able to query other collections than the
+ iris collection).
+ Most methods convert resulting documents from the BSON format (MongoDB) to JSON. This mainly avoids the issue of
+ ObjectId type (for MongoDB '_id' field).
+
+ The constructor initializes the logger and it automatically connects to the database.
+ The name of the database and of the three collections
+ are based on the default names (the ones from the dump). If the names are changed in MongoDB, they should be changed
+ in the `config.py` file.
+
+ Examples of usages, including testing geospatial queries, are available in `tests/api_tests.py`.
+
+ An example of IRIS following the GeoJSON format is provided in `data/example-iris.json`.
+
+ Additional resources:
+
+ - [MongoDB documentation](http://www.mongodb.org/)
+
+ - [GeoJSON format specifications](https://geojson.org/)
+
+ - [pymongo API](http://api.mongodb.com/python/current/) between Python and MongoDB
+
+ """
+
+ def __init__(self):
+ logging.basicConfig(format='[%(levelname)s] - %(name)s - %(asctime)s : %(message)s')
+ self.logger = logging.getLogger()
+ self.logger.setLevel(logging.INFO)
+ self.connection, self.connection_status = self.init_connection() # default connection on 'localhost', 27017
+ self.database = self.connection[config.database_iris] # database for HiL project
+ self.collection_iris = self.database[config.collection_iris]
+ self.collection_indic = self.database[config.collection_indic]
+ self.collection_sources = self.database[config.collection_sources]
+
+ @staticmethod
+ def bson_to_json(doc_bson):
+ """
+ Converts the bson data to valid JSON (including the ObjectId type converted to {"$oid": <string>}).
+
+ Args:
+ doc_bson: the BSON data to be converted
+
+ Returns:
+ doc_json: a JSON object
+ """
+ doc_json = json.loads(json_util.dumps(doc_bson, json_options=json_util.RELAXED_JSON_OPTIONS))
+ return doc_json
+
+ def init_connection(self):
+ """
+ Tries to connect to MongoDB. The output connection object do not provide reliable connection status, only the
+ boolean connection_status indicates whether the connection is working or not.
+
+ Returns:
+ connection: an object with information about the connection
+ connection_status: a boolean indicating whether the connection is a success or a failure
+ """
+ connection_status = True # by default, connection is ok
+ connection = None
+ try:
+ connection = pymongo.MongoClient(serverSelectionTimeoutMS=config.max_timeout) # default localhost:27017
+ connection.server_info() # forces a query to MongoDB (for checking the connection)
+ except pymongo.errors.ServerSelectionTimeoutError as e:
+ self.logger.error('Could not connect to the MongoDB database ! Have you launched MongoDB ? ' + str(e))
+ connection_status = False
+ return connection, connection_status
+
+ @staticmethod
+ def _parse_json_to_dict(json_file_path):
+ """ Converts a JSON file denoted by json_file_path into a Python dictionary. """
+ with open(json_file_path) as data_file:
+ data = json.load(data_file)
+ data_file.close()
+ return data
+
+ @staticmethod
+ def _save_dict_to_json(json_file_path, dict_geo):
+ """ Converts and saves a Python dictionary into a JSON file denoted by json_file_path. """
+ with open(json_file_path, 'w') as data_file:
+ json.dump(dict_geo, data_file)
+
+ def create_index(self, iris_collection):
+ """ Rebuilds a geospatial index on the iris collection. Only used in case of restoration/import. """
+ self.logger.info("Creating index on 'geometry' using " + pymongo.GEOSPHERE)
+ iris_collection.create_index([("geometry", pymongo.GEOSPHERE)])
+ self.logger.info("Index created")
+
+ def count_documents(self, collection, json_query):
+ """
+ Counts the number of documents that satisfy json_query in the given collection.
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+
+ Returns:
+ count: an integer representing the number of documents
+ """
+ return collection.count_documents(json_query)
+
+ def find_one_document(self, collection, json_query):
+ """
+ Finds the first document in the given collection that satisfies json_query.
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+
+ Returns:
+ doc_json: a dictionary representing an iris, or None
+ """
+ doc = collection.find_one(json_query)
+ doc_json = Mongiris.bson_to_json(doc)
+ return doc_json
+
+ def get_iris_from_code(self, code_iris):
+ """
+ Returns the iris identified by the given code_iris.
+
+ Args:
+ code_iris: a string containing the code of the searched iris
+
+ Returns:
+ iris: a dictionary representing an iris, or None
+ """
+ iris = self.find_one_document(self.collection_iris, {"properties.CODE_IRIS": code_iris})
+ return iris
+
+ def find_documents(self, collection, json_query, json_projection=None):
+ """
+ Finds the first document in the given collection that satisfies json_query.
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ cursor = collection.find(json_query, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json
+
+ def get_random_document(self, collection):
+ """ Returns a random document from the given collection. """
+ random_iris = collection.aggregate([{"$sample": {"size": 1}}]).next()
+ doc_json = Mongiris.bson_to_json(random_iris)
+ return doc_json
+
+ def update_one_document(self, collection, json_query, json_updates):
+ """
+ Updates the first document satisfying json_query by setting new values from json_updates.
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+ json_updates: a json document containing values to be updates (using $set operator)
+
+ Returns:
+ json_result: an UpdateResult json document containing information about the operation
+ """
+ json_result = collection.update_one(json_query, json_updates)
+ return json_result
+
+ def replace_one_document(self, collection, json_query, json_replace_doc, upsert=False):
+ """
+ Replaces the first document satisfying json_query by the document json_replace_doc (their _id are identical).
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+ json_replace_doc: the replacement doc (if _id set, should be the same _id as the doc matching json_query)
+ upsert: a boolean, whether the json_replace_doc should be inserted if no document match json_query
+
+ Returns:
+ json_result: an UpdateResult json document containing information about the operation
+ """
+ json_result = collection.replace_one(json_query, json_replace_doc, upsert)
+ return json_result
+
+ def insert_one_document(self, collection, doc):
+ """
+ Inserts a new document in the collection.
+
+ Args:
+ collection: a string representing the collection name
+ doc: a dict representing the document to be added
+
+ Returns:
+ json_result: an InsertOneResult json document containing information about the operation
+ """
+ json_result = collection.insert_one(doc)
+ return json_result # eg, the new _id is in json_result.inserted_id
+
+ def delete_all(self, collection):
+ """
+ Deletes all documents in the collection.
+
+ Args:
+ collection: a string representing the collection name
+
+ Returns:
+ json_result: an DeleteResult json document containing information about the operation
+ """
+ json_result = collection.delete_many({}) # empty collection
+ return json_result
+
+ def geo_within(self, collection, geometry, json_projection=None):
+ """
+ Finds all documents from given collection and which contain totally the given geometry.
+ Cannot be used to find the IRIS containing a point (geometry must be a polygon).
+
+ Args:
+ collection: a string representing the collection name
+ geometry: a geojson geometry ("Polygon", "$box" or "MultiPolygon", NO "Point")
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ cursor = self.find_documents(collection, {"geometry": {"$geoWithin": {"$geometry": geometry}}}, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json
+
+ def geo_within_sphere(self, collection, sphere, json_projection=None):
+ """
+ Finds all documents from given collection and which contain totally the given sphere.
+ Cannot be used to find the IRIS containing a point (geometry must be a polygon, with min. 3 points).
+
+ Args:
+ collection: a string representing the collection name
+ sphere: a geojson geometry defined by a center and a radius in radians
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ cursor = self.find_documents(collection, {"geometry": {"$geoWithin": sphere}}, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json
+
+ def intersect(self, collection, geometry, json_projection=None):
+ """
+ Finds all documents from given collection and which intersect the given geometry.
+
+ Args:
+ collection: a string representing the collection name
+ geometry: a geojson geometry
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ cursor = self.find_documents(collection, {"geometry": {"$geoIntersects": {
+ "$geometry": geometry}}}, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json
+
+ @staticmethod
+ def get_geojson_point(coordinates):
+ """
+ Builds a dictionary with GeoJSON syntax for a point using the given coordinates.
+
+ Args:
+ coordinates: the coordinates (long, lat) as a list, e.g. [4.8, 45.7]
+
+ Returns:
+ point: a dictionary with GeoJSON syntax for a Point
+ """
+ return {"type": "Point", "coordinates": coordinates}
+
+ @staticmethod
+ def convert_geojson_box_to_polygon(lng1, lat1, lng2, lat2):
+ """
+ Builds a dictionary with GeoJSON syntax for a polygon using two coordinates (points south-west and north-east).
+ This method builds the polygon by adding 2 missing points (north-west and south-east) and adds the starting
+ point (south-west) to end the loop.
+ The MongoDB $box operator is not supported with 2d-spherical indexes.
+
+ Args:
+ lng1: longitude of the first point (south-west) of the box
+ lat1: latitude of the first point (south-west) of the box
+ lng2: longitude of the second point (north-east) of the box
+ lat2: latitude of the second point (north-east) of the box
+
+ Returns:
+ box: a dictionary with GeoJSON syntax for a Box
+ """
+ coordinates = [[[lng1, lat1], [lng1, lat2], [lng2, lat2], [lng2, lat1], [lng1, lat1]]]
+ return Mongiris.get_geojson_polygon(coordinates)
+
+ @staticmethod
+ def get_geojson_polygon(coordinates):
+ """
+ Builds a dictionary with GeoJSON syntax for a polygon using the given coordinates.
+ Careful: polygons must be closed ! (first coordinate must be identical to last coordinate)
+
+ Args:
+ coordinates: the coordinates (long, lat) as a list of list, e.g. [[[4.8, 45.7], [4.9, 47.8]]]
+
+ Returns:
+ polygon: a dictionary with GeoJSON syntax for a Polygon
+ """
+ return {"type": "Polygon", "coordinates": coordinates}
+
+ def point_in_which_iris(self, coordinates, json_projection=None):
+ """
+ Finds the document (IRIS) containing the given coordinates. Uses near() since geo_within() requires a Polygon.
+ Careful: the near() operator may return several iris (low probability since distance = 1 meter) and only the
+ first one is returned.
+
+ Args:
+ coordinates: an array of coordinates (long, lat)
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a json document or None
+ """
+ results = self.near(self.collection_iris, coordinates, json_projection, 1) # distance = 1 meter
+ if len(results) == 0:
+ return None
+ return results[0]
+
+ def near(self, collection, coordinates, json_projection=None, distance_max=2000):
+ """
+ Finds all documents from given collection and which are near the given geometry (according to distance_max).
+
+ Args:
+ collection: a string representing the collection name
+ coordinates: an array of coordinates (long, lat) - near only accepts Point
+ json_projection: a json document indicating the fields that appear in the results
+ distance_max: the maximum distance of resulting iris, in meters
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ geometry = Mongiris.get_geojson_point(coordinates)
+ cursor = self.find_documents(collection, {"geometry": {"$near": {
+ "$geometry": geometry, "$maxDistance": distance_max}}}, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json
+
+ @staticmethod
+ def adjacent(collection, geometry, json_projection=None, distance=20, exclude_geometry_iris=None):
+ """
+ Finds all adjacent neighbors of an iris represented by geometry.
+ No adjacent function, so use all coordinates of an iris and find the closest iris (according to distance).
+ Could be done directly with near(), but near() is less accurate and thus incomplete.
+
+ Args:
+ collection: a string representing the collection name
+ geometry: a geojson geometry (Point, Polygon, etc.)
+ json_projection: a json document indicating the fields that appear in the results
+ distance: the maximum distance for an adjacent neighbour, in meters (10 to 50 meters are fine)
+ exclude_geometry_iris: the document _id of the iris represented by geometry, if it needs to be excluded
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ results = list()
+ results_ids = list()
+ if exclude_geometry_iris is not None: # to exclude the iris represented by geometry, add it to the results ids
+ results_ids.append(exclude_geometry_iris)
+ for coords in geometry["coordinates"][0]:
+ geometry_coords = Mongiris.get_geojson_point(coords)
+ cursor = collection.find({"geometry": {"$near": {"$geometry": geometry_coords,
+ "$maxDistance": distance}}}, json_projection)
+ for doc in cursor:
+ doc_id = doc["_id"]
+ if doc_id not in results_ids: # add the new adjacent iris if not already in the results
+ results.append(doc)
+ results_ids.append(doc_id)
+ doc_json = Mongiris.bson_to_json(results)
+ return doc_json</code></pre>
+</details>
+<h3>Static methods</h3>
+<dl>
+<dt id="api.Mongiris.adjacent"><code class="name flex">
+<span>def <span class="ident">adjacent</span></span>(<span>collection, geometry, json_projection=None, distance=20, exclude_geometry_iris=None)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Finds all adjacent neighbors of an iris represented by geometry.
+No adjacent function, so use all coordinates of an iris and find the closest iris (according to distance).
+Could be done directly with near(), but near() is less accurate and thus incomplete.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>collection</code></strong></dt>
+<dd>a string representing the collection name</dd>
+<dt><strong><code>geometry</code></strong></dt>
+<dd>a geojson geometry (Point, Polygon, etc.)</dd>
+<dt><strong><code>json_projection</code></strong></dt>
+<dd>a json document indicating the fields that appear in the results</dd>
+<dt><strong><code>distance</code></strong></dt>
+<dd>the maximum distance for an adjacent neighbour, in meters (10 to 50 meters are fine)</dd>
+<dt><strong><code>exclude_geometry_iris</code></strong></dt>
+<dd>the document _id of the iris represented by geometry, if it needs to be excluded</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>doc_json</code></strong></dt>
+<dd>a cursor (set of documents)</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">@staticmethod
+def adjacent(collection, geometry, json_projection=None, distance=20, exclude_geometry_iris=None):
+ """
+ Finds all adjacent neighbors of an iris represented by geometry.
+ No adjacent function, so use all coordinates of an iris and find the closest iris (according to distance).
+ Could be done directly with near(), but near() is less accurate and thus incomplete.
+
+ Args:
+ collection: a string representing the collection name
+ geometry: a geojson geometry (Point, Polygon, etc.)
+ json_projection: a json document indicating the fields that appear in the results
+ distance: the maximum distance for an adjacent neighbour, in meters (10 to 50 meters are fine)
+ exclude_geometry_iris: the document _id of the iris represented by geometry, if it needs to be excluded
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ results = list()
+ results_ids = list()
+ if exclude_geometry_iris is not None: # to exclude the iris represented by geometry, add it to the results ids
+ results_ids.append(exclude_geometry_iris)
+ for coords in geometry["coordinates"][0]:
+ geometry_coords = Mongiris.get_geojson_point(coords)
+ cursor = collection.find({"geometry": {"$near": {"$geometry": geometry_coords,
+ "$maxDistance": distance}}}, json_projection)
+ for doc in cursor:
+ doc_id = doc["_id"]
+ if doc_id not in results_ids: # add the new adjacent iris if not already in the results
+ results.append(doc)
+ results_ids.append(doc_id)
+ doc_json = Mongiris.bson_to_json(results)
+ return doc_json</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.bson_to_json"><code class="name flex">
+<span>def <span class="ident">bson_to_json</span></span>(<span>doc_bson)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Converts the bson data to valid JSON (including the ObjectId type converted to {"$oid": <string>}).</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>doc_bson</code></strong></dt>
+<dd>the BSON data to be converted</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>doc_json</code></strong></dt>
+<dd>a JSON object</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">@staticmethod
+def bson_to_json(doc_bson):
+ """
+ Converts the bson data to valid JSON (including the ObjectId type converted to {"$oid": <string>}).
+
+ Args:
+ doc_bson: the BSON data to be converted
+
+ Returns:
+ doc_json: a JSON object
+ """
+ doc_json = json.loads(json_util.dumps(doc_bson, json_options=json_util.RELAXED_JSON_OPTIONS))
+ return doc_json</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.convert_geojson_box_to_polygon"><code class="name flex">
+<span>def <span class="ident">convert_geojson_box_to_polygon</span></span>(<span>lng1, lat1, lng2, lat2)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Builds a dictionary with GeoJSON syntax for a polygon using two coordinates (points south-west and north-east).
+This method builds the polygon by adding 2 missing points (north-west and south-east) and adds the starting
+point (south-west) to end the loop.
+The MongoDB $box operator is not supported with 2d-spherical indexes.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>lng1</code></strong></dt>
+<dd>longitude of the first point (south-west) of the box</dd>
+<dt><strong><code>lat1</code></strong></dt>
+<dd>latitude of the first point (south-west) of the box</dd>
+<dt><strong><code>lng2</code></strong></dt>
+<dd>longitude of the second point (north-east) of the box</dd>
+<dt><strong><code>lat2</code></strong></dt>
+<dd>latitude of the second point (north-east) of the box</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>box</code></strong></dt>
+<dd>a dictionary with GeoJSON syntax for a Box</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">@staticmethod
+def convert_geojson_box_to_polygon(lng1, lat1, lng2, lat2):
+ """
+ Builds a dictionary with GeoJSON syntax for a polygon using two coordinates (points south-west and north-east).
+ This method builds the polygon by adding 2 missing points (north-west and south-east) and adds the starting
+ point (south-west) to end the loop.
+ The MongoDB $box operator is not supported with 2d-spherical indexes.
+
+ Args:
+ lng1: longitude of the first point (south-west) of the box
+ lat1: latitude of the first point (south-west) of the box
+ lng2: longitude of the second point (north-east) of the box
+ lat2: latitude of the second point (north-east) of the box
+
+ Returns:
+ box: a dictionary with GeoJSON syntax for a Box
+ """
+ coordinates = [[[lng1, lat1], [lng1, lat2], [lng2, lat2], [lng2, lat1], [lng1, lat1]]]
+ return Mongiris.get_geojson_polygon(coordinates)</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.get_geojson_point"><code class="name flex">
+<span>def <span class="ident">get_geojson_point</span></span>(<span>coordinates)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Builds a dictionary with GeoJSON syntax for a point using the given coordinates.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>coordinates</code></strong></dt>
+<dd>the coordinates (long, lat) as a list, e.g. [4.8, 45.7]</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>point</code></strong></dt>
+<dd>a dictionary with GeoJSON syntax for a Point</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">@staticmethod
+def get_geojson_point(coordinates):
+ """
+ Builds a dictionary with GeoJSON syntax for a point using the given coordinates.
+
+ Args:
+ coordinates: the coordinates (long, lat) as a list, e.g. [4.8, 45.7]
+
+ Returns:
+ point: a dictionary with GeoJSON syntax for a Point
+ """
+ return {"type": "Point", "coordinates": coordinates}</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.get_geojson_polygon"><code class="name flex">
+<span>def <span class="ident">get_geojson_polygon</span></span>(<span>coordinates)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Builds a dictionary with GeoJSON syntax for a polygon using the given coordinates.
+Careful: polygons must be closed ! (first coordinate must be identical to last coordinate)</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>coordinates</code></strong></dt>
+<dd>the coordinates (long, lat) as a list of list, e.g. [[[4.8, 45.7], [4.9, 47.8]]]</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>polygon</code></strong></dt>
+<dd>a dictionary with GeoJSON syntax for a Polygon</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">@staticmethod
+def get_geojson_polygon(coordinates):
+ """
+ Builds a dictionary with GeoJSON syntax for a polygon using the given coordinates.
+ Careful: polygons must be closed ! (first coordinate must be identical to last coordinate)
+
+ Args:
+ coordinates: the coordinates (long, lat) as a list of list, e.g. [[[4.8, 45.7], [4.9, 47.8]]]
+
+ Returns:
+ polygon: a dictionary with GeoJSON syntax for a Polygon
+ """
+ return {"type": "Polygon", "coordinates": coordinates}</code></pre>
+</details>
+</dd>
+</dl>
+<h3>Methods</h3>
+<dl>
+<dt id="api.Mongiris.count_documents"><code class="name flex">
+<span>def <span class="ident">count_documents</span></span>(<span>self, collection, json_query)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Counts the number of documents that satisfy json_query in the given collection.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>collection</code></strong></dt>
+<dd>a string representing the collection name</dd>
+<dt><strong><code>json_query</code></strong></dt>
+<dd>a dict containing the query criteria (<a href="https://docs.mongodb.com/manual/tutorial/query-documents/">https://docs.mongodb.com/manual/tutorial/query-documents/</a>)</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>count</code></strong></dt>
+<dd>an integer representing the number of documents</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def count_documents(self, collection, json_query):
+ """
+ Counts the number of documents that satisfy json_query in the given collection.
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+
+ Returns:
+ count: an integer representing the number of documents
+ """
+ return collection.count_documents(json_query)</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.create_index"><code class="name flex">
+<span>def <span class="ident">create_index</span></span>(<span>self, iris_collection)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Rebuilds a geospatial index on the iris collection. Only used in case of restoration/import.</p></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def create_index(self, iris_collection):
+ """ Rebuilds a geospatial index on the iris collection. Only used in case of restoration/import. """
+ self.logger.info("Creating index on 'geometry' using " + pymongo.GEOSPHERE)
+ iris_collection.create_index([("geometry", pymongo.GEOSPHERE)])
+ self.logger.info("Index created")</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.delete_all"><code class="name flex">
+<span>def <span class="ident">delete_all</span></span>(<span>self, collection)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Deletes all documents in the collection.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>collection</code></strong></dt>
+<dd>a string representing the collection name</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>json_result</code></strong></dt>
+<dd>an DeleteResult json document containing information about the operation</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def delete_all(self, collection):
+ """
+ Deletes all documents in the collection.
+
+ Args:
+ collection: a string representing the collection name
+
+ Returns:
+ json_result: an DeleteResult json document containing information about the operation
+ """
+ json_result = collection.delete_many({}) # empty collection
+ return json_result</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.find_documents"><code class="name flex">
+<span>def <span class="ident">find_documents</span></span>(<span>self, collection, json_query, json_projection=None)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Finds the first document in the given collection that satisfies json_query.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>collection</code></strong></dt>
+<dd>a string representing the collection name</dd>
+<dt><strong><code>json_query</code></strong></dt>
+<dd>a dict containing the query criteria (<a href="https://docs.mongodb.com/manual/tutorial/query-documents/">https://docs.mongodb.com/manual/tutorial/query-documents/</a>)</dd>
+<dt><strong><code>json_projection</code></strong></dt>
+<dd>a json document indicating the fields that appear in the results</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>doc_json</code></strong></dt>
+<dd>a cursor (set of documents)</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def find_documents(self, collection, json_query, json_projection=None):
+ """
+ Finds the first document in the given collection that satisfies json_query.
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ cursor = collection.find(json_query, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.find_one_document"><code class="name flex">
+<span>def <span class="ident">find_one_document</span></span>(<span>self, collection, json_query)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Finds the first document in the given collection that satisfies json_query.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>collection</code></strong></dt>
+<dd>a string representing the collection name</dd>
+<dt><strong><code>json_query</code></strong></dt>
+<dd>a dict containing the query criteria (<a href="https://docs.mongodb.com/manual/tutorial/query-documents/">https://docs.mongodb.com/manual/tutorial/query-documents/</a>)</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>doc_json</code></strong></dt>
+<dd>a dictionary representing an iris, or None</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def find_one_document(self, collection, json_query):
+ """
+ Finds the first document in the given collection that satisfies json_query.
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+
+ Returns:
+ doc_json: a dictionary representing an iris, or None
+ """
+ doc = collection.find_one(json_query)
+ doc_json = Mongiris.bson_to_json(doc)
+ return doc_json</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.geo_within"><code class="name flex">
+<span>def <span class="ident">geo_within</span></span>(<span>self, collection, geometry, json_projection=None)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Finds all documents from given collection and which contain totally the given geometry.
+Cannot be used to find the IRIS containing a point (geometry must be a polygon).</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>collection</code></strong></dt>
+<dd>a string representing the collection name</dd>
+<dt><strong><code>geometry</code></strong></dt>
+<dd>a geojson geometry ("Polygon", "$box" or "MultiPolygon", NO "Point")</dd>
+<dt><strong><code>json_projection</code></strong></dt>
+<dd>a json document indicating the fields that appear in the results</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>doc_json</code></strong></dt>
+<dd>a cursor (set of documents)</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def geo_within(self, collection, geometry, json_projection=None):
+ """
+ Finds all documents from given collection and which contain totally the given geometry.
+ Cannot be used to find the IRIS containing a point (geometry must be a polygon).
+
+ Args:
+ collection: a string representing the collection name
+ geometry: a geojson geometry ("Polygon", "$box" or "MultiPolygon", NO "Point")
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ cursor = self.find_documents(collection, {"geometry": {"$geoWithin": {"$geometry": geometry}}}, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.geo_within_sphere"><code class="name flex">
+<span>def <span class="ident">geo_within_sphere</span></span>(<span>self, collection, sphere, json_projection=None)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Finds all documents from given collection and which contain totally the given sphere.
+Cannot be used to find the IRIS containing a point (geometry must be a polygon, with min. 3 points).</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>collection</code></strong></dt>
+<dd>a string representing the collection name</dd>
+<dt><strong><code>sphere</code></strong></dt>
+<dd>a geojson geometry defined by a center and a radius in radians</dd>
+<dt><strong><code>json_projection</code></strong></dt>
+<dd>a json document indicating the fields that appear in the results</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>doc_json</code></strong></dt>
+<dd>a cursor (set of documents)</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def geo_within_sphere(self, collection, sphere, json_projection=None):
+ """
+ Finds all documents from given collection and which contain totally the given sphere.
+ Cannot be used to find the IRIS containing a point (geometry must be a polygon, with min. 3 points).
+
+ Args:
+ collection: a string representing the collection name
+ sphere: a geojson geometry defined by a center and a radius in radians
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ cursor = self.find_documents(collection, {"geometry": {"$geoWithin": sphere}}, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.get_iris_from_code"><code class="name flex">
+<span>def <span class="ident">get_iris_from_code</span></span>(<span>self, code_iris)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Returns the iris identified by the given code_iris.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>code_iris</code></strong></dt>
+<dd>a string containing the code of the searched iris</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>iris</code></strong></dt>
+<dd>a dictionary representing an iris, or None</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def get_iris_from_code(self, code_iris):
+ """
+ Returns the iris identified by the given code_iris.
+
+ Args:
+ code_iris: a string containing the code of the searched iris
+
+ Returns:
+ iris: a dictionary representing an iris, or None
+ """
+ iris = self.find_one_document(self.collection_iris, {"properties.CODE_IRIS": code_iris})
+ return iris</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.get_random_document"><code class="name flex">
+<span>def <span class="ident">get_random_document</span></span>(<span>self, collection)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Returns a random document from the given collection.</p></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def get_random_document(self, collection):
+ """ Returns a random document from the given collection. """
+ random_iris = collection.aggregate([{"$sample": {"size": 1}}]).next()
+ doc_json = Mongiris.bson_to_json(random_iris)
+ return doc_json</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.init_connection"><code class="name flex">
+<span>def <span class="ident">init_connection</span></span>(<span>self)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Tries to connect to MongoDB. The output connection object do not provide reliable connection status, only the
+boolean connection_status indicates whether the connection is working or not.</p>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>connection</code></strong></dt>
+<dd>an object with information about the connection</dd>
+<dt><strong><code>connection_status</code></strong></dt>
+<dd>a boolean indicating whether the connection is a success or a failure</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def init_connection(self):
+ """
+ Tries to connect to MongoDB. The output connection object do not provide reliable connection status, only the
+ boolean connection_status indicates whether the connection is working or not.
+
+ Returns:
+ connection: an object with information about the connection
+ connection_status: a boolean indicating whether the connection is a success or a failure
+ """
+ connection_status = True # by default, connection is ok
+ connection = None
+ try:
+ connection = pymongo.MongoClient(serverSelectionTimeoutMS=config.max_timeout) # default localhost:27017
+ connection.server_info() # forces a query to MongoDB (for checking the connection)
+ except pymongo.errors.ServerSelectionTimeoutError as e:
+ self.logger.error('Could not connect to the MongoDB database ! Have you launched MongoDB ? ' + str(e))
+ connection_status = False
+ return connection, connection_status</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.insert_one_document"><code class="name flex">
+<span>def <span class="ident">insert_one_document</span></span>(<span>self, collection, doc)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Inserts a new document in the collection.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>collection</code></strong></dt>
+<dd>a string representing the collection name</dd>
+<dt><strong><code>doc</code></strong></dt>
+<dd>a dict representing the document to be added</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>json_result</code></strong></dt>
+<dd>an InsertOneResult json document containing information about the operation</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def insert_one_document(self, collection, doc):
+ """
+ Inserts a new document in the collection.
+
+ Args:
+ collection: a string representing the collection name
+ doc: a dict representing the document to be added
+
+ Returns:
+ json_result: an InsertOneResult json document containing information about the operation
+ """
+ json_result = collection.insert_one(doc)
+ return json_result # eg, the new _id is in json_result.inserted_id</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.intersect"><code class="name flex">
+<span>def <span class="ident">intersect</span></span>(<span>self, collection, geometry, json_projection=None)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Finds all documents from given collection and which intersect the given geometry.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>collection</code></strong></dt>
+<dd>a string representing the collection name</dd>
+<dt><strong><code>geometry</code></strong></dt>
+<dd>a geojson geometry</dd>
+<dt><strong><code>json_projection</code></strong></dt>
+<dd>a json document indicating the fields that appear in the results</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>doc_json</code></strong></dt>
+<dd>a cursor (set of documents)</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def intersect(self, collection, geometry, json_projection=None):
+ """
+ Finds all documents from given collection and which intersect the given geometry.
+
+ Args:
+ collection: a string representing the collection name
+ geometry: a geojson geometry
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ cursor = self.find_documents(collection, {"geometry": {"$geoIntersects": {
+ "$geometry": geometry}}}, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.near"><code class="name flex">
+<span>def <span class="ident">near</span></span>(<span>self, collection, coordinates, json_projection=None, distance_max=2000)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Finds all documents from given collection and which are near the given geometry (according to distance_max).</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>collection</code></strong></dt>
+<dd>a string representing the collection name</dd>
+<dt><strong><code>coordinates</code></strong></dt>
+<dd>an array of coordinates (long, lat) - near only accepts Point</dd>
+<dt><strong><code>json_projection</code></strong></dt>
+<dd>a json document indicating the fields that appear in the results</dd>
+<dt><strong><code>distance_max</code></strong></dt>
+<dd>the maximum distance of resulting iris, in meters</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>doc_json</code></strong></dt>
+<dd>a cursor (set of documents)</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def near(self, collection, coordinates, json_projection=None, distance_max=2000):
+ """
+ Finds all documents from given collection and which are near the given geometry (according to distance_max).
+
+ Args:
+ collection: a string representing the collection name
+ coordinates: an array of coordinates (long, lat) - near only accepts Point
+ json_projection: a json document indicating the fields that appear in the results
+ distance_max: the maximum distance of resulting iris, in meters
+
+ Returns:
+ doc_json: a cursor (set of documents)
+ """
+ geometry = Mongiris.get_geojson_point(coordinates)
+ cursor = self.find_documents(collection, {"geometry": {"$near": {
+ "$geometry": geometry, "$maxDistance": distance_max}}}, json_projection)
+ doc_json = Mongiris.bson_to_json(cursor)
+ return doc_json</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.point_in_which_iris"><code class="name flex">
+<span>def <span class="ident">point_in_which_iris</span></span>(<span>self, coordinates, json_projection=None)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Finds the document (IRIS) containing the given coordinates. Uses near() since geo_within() requires a Polygon.
+Careful: the near() operator may return several iris (low probability since distance = 1 meter) and only the
+first one is returned.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>coordinates</code></strong></dt>
+<dd>an array of coordinates (long, lat)</dd>
+<dt><strong><code>json_projection</code></strong></dt>
+<dd>a json document indicating the fields that appear in the results</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>doc_json</code></strong></dt>
+<dd>a json document or None</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def point_in_which_iris(self, coordinates, json_projection=None):
+ """
+ Finds the document (IRIS) containing the given coordinates. Uses near() since geo_within() requires a Polygon.
+ Careful: the near() operator may return several iris (low probability since distance = 1 meter) and only the
+ first one is returned.
+
+ Args:
+ coordinates: an array of coordinates (long, lat)
+ json_projection: a json document indicating the fields that appear in the results
+
+ Returns:
+ doc_json: a json document or None
+ """
+ results = self.near(self.collection_iris, coordinates, json_projection, 1) # distance = 1 meter
+ if len(results) == 0:
+ return None
+ return results[0]</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.replace_one_document"><code class="name flex">
+<span>def <span class="ident">replace_one_document</span></span>(<span>self, collection, json_query, json_replace_doc, upsert=False)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Replaces the first document satisfying json_query by the document json_replace_doc (their _id are identical).</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>collection</code></strong></dt>
+<dd>a string representing the collection name</dd>
+<dt><strong><code>json_query</code></strong></dt>
+<dd>a dict containing the query criteria (<a href="https://docs.mongodb.com/manual/tutorial/query-documents/">https://docs.mongodb.com/manual/tutorial/query-documents/</a>)</dd>
+<dt><strong><code>json_replace_doc</code></strong></dt>
+<dd>the replacement doc (if _id set, should be the same _id as the doc matching json_query)</dd>
+<dt><strong><code>upsert</code></strong></dt>
+<dd>a boolean, whether the json_replace_doc should be inserted if no document match json_query</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>json_result</code></strong></dt>
+<dd>an UpdateResult json document containing information about the operation</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def replace_one_document(self, collection, json_query, json_replace_doc, upsert=False):
+ """
+ Replaces the first document satisfying json_query by the document json_replace_doc (their _id are identical).
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+ json_replace_doc: the replacement doc (if _id set, should be the same _id as the doc matching json_query)
+ upsert: a boolean, whether the json_replace_doc should be inserted if no document match json_query
+
+ Returns:
+ json_result: an UpdateResult json document containing information about the operation
+ """
+ json_result = collection.replace_one(json_query, json_replace_doc, upsert)
+ return json_result</code></pre>
+</details>
+</dd>
+<dt id="api.Mongiris.update_one_document"><code class="name flex">
+<span>def <span class="ident">update_one_document</span></span>(<span>self, collection, json_query, json_updates)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Updates the first document satisfying json_query by setting new values from json_updates.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>collection</code></strong></dt>
+<dd>a string representing the collection name</dd>
+<dt><strong><code>json_query</code></strong></dt>
+<dd>a dict containing the query criteria (<a href="https://docs.mongodb.com/manual/tutorial/query-documents/">https://docs.mongodb.com/manual/tutorial/query-documents/</a>)</dd>
+<dt><strong><code>json_updates</code></strong></dt>
+<dd>a json document containing values to be updates (using $set operator)</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>json_result</code></strong></dt>
+<dd>an UpdateResult json document containing information about the operation</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def update_one_document(self, collection, json_query, json_updates):
+ """
+ Updates the first document satisfying json_query by setting new values from json_updates.
+
+ Args:
+ collection: a string representing the collection name
+ json_query: a dict containing the query criteria (https://docs.mongodb.com/manual/tutorial/query-documents/)
+ json_updates: a json document containing values to be updates (using $set operator)
+
+ Returns:
+ json_result: an UpdateResult json document containing information about the operation
+ """
+ json_result = collection.update_one(json_query, json_updates)
+ return json_result</code></pre>
+</details>
+</dd>
+</dl>
+</dd>
+</dl>
+</section>
+</article>
+<nav id="sidebar">
+<h1>Index</h1>
+<div class="toc">
+<ul></ul>
+</div>
+<ul id="index">
+<li><h3><a href="#header-classes">Classes</a></h3>
+<ul>
+<li>
+<h4><code><a title="api.Mongiris" href="#api.Mongiris">Mongiris</a></code></h4>
+<ul class="">
+<li><code><a title="api.Mongiris.adjacent" href="#api.Mongiris.adjacent">adjacent</a></code></li>
+<li><code><a title="api.Mongiris.bson_to_json" href="#api.Mongiris.bson_to_json">bson_to_json</a></code></li>
+<li><code><a title="api.Mongiris.convert_geojson_box_to_polygon" href="#api.Mongiris.convert_geojson_box_to_polygon">convert_geojson_box_to_polygon</a></code></li>
+<li><code><a title="api.Mongiris.count_documents" href="#api.Mongiris.count_documents">count_documents</a></code></li>
+<li><code><a title="api.Mongiris.create_index" href="#api.Mongiris.create_index">create_index</a></code></li>
+<li><code><a title="api.Mongiris.delete_all" href="#api.Mongiris.delete_all">delete_all</a></code></li>
+<li><code><a title="api.Mongiris.find_documents" href="#api.Mongiris.find_documents">find_documents</a></code></li>
+<li><code><a title="api.Mongiris.find_one_document" href="#api.Mongiris.find_one_document">find_one_document</a></code></li>
+<li><code><a title="api.Mongiris.geo_within" href="#api.Mongiris.geo_within">geo_within</a></code></li>
+<li><code><a title="api.Mongiris.geo_within_sphere" href="#api.Mongiris.geo_within_sphere">geo_within_sphere</a></code></li>
+<li><code><a title="api.Mongiris.get_geojson_point" href="#api.Mongiris.get_geojson_point">get_geojson_point</a></code></li>
+<li><code><a title="api.Mongiris.get_geojson_polygon" href="#api.Mongiris.get_geojson_polygon">get_geojson_polygon</a></code></li>
+<li><code><a title="api.Mongiris.get_iris_from_code" href="#api.Mongiris.get_iris_from_code">get_iris_from_code</a></code></li>
+<li><code><a title="api.Mongiris.get_random_document" href="#api.Mongiris.get_random_document">get_random_document</a></code></li>
+<li><code><a title="api.Mongiris.init_connection" href="#api.Mongiris.init_connection">init_connection</a></code></li>
+<li><code><a title="api.Mongiris.insert_one_document" href="#api.Mongiris.insert_one_document">insert_one_document</a></code></li>
+<li><code><a title="api.Mongiris.intersect" href="#api.Mongiris.intersect">intersect</a></code></li>
+<li><code><a title="api.Mongiris.near" href="#api.Mongiris.near">near</a></code></li>
+<li><code><a title="api.Mongiris.point_in_which_iris" href="#api.Mongiris.point_in_which_iris">point_in_which_iris</a></code></li>
+<li><code><a title="api.Mongiris.replace_one_document" href="#api.Mongiris.replace_one_document">replace_one_document</a></code></li>
+<li><code><a title="api.Mongiris.update_one_document" href="#api.Mongiris.update_one_document">update_one_document</a></code></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</nav>
+</main>
+<footer id="footer">
+<p>Generated by <a href="https://pdoc3.github.io/pdoc"><cite>pdoc</cite> 0.6.3</a>.</p>
+</footer>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+<script>hljs.initHighlightingOnLoad()</script>
+</body>
+</html>
\ No newline at end of file
diff --git a/html/index.html b/html/index.html
new file mode 100644
index 0000000000000000000000000000000000000000..ac31c6ba909be5dc2211cacf0e4fc298242f1b6d
--- /dev/null
+++ b/html/index.html
@@ -0,0 +1,56 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1" />
+<meta name="generator" content="pdoc 0.6.3" />
+<title>Package mongiris documentation</title>
+<meta name="description" content="" />
+<link href='https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.0/normalize.min.css' rel='stylesheet'>
+<link href='https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/8.0.0/sanitize.min.css' rel='stylesheet'>
+<link href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css" rel="stylesheet">
+<style>.flex{display:flex !important}body{line-height:1.5em}#content{padding:20px}#sidebar{padding:30px;overflow:hidden}.http-server-breadcrumbs{font-size:130%;margin:0 0 15px 0}#footer{font-size:.75em;padding:5px 30px;border-top:1px solid #ddd;text-align:right}#footer p{margin:0 0 0 1em;display:inline-block}#footer p:last-child{margin-right:30px}h1,h2,h3,h4,h5{font-weight:300}h1{font-size:2.5em;line-height:1.1em}h2{font-size:1.75em;margin:1em 0 .50em 0}h3{font-size:1.4em;margin:25px 0 10px 0}h4{margin:0;font-size:105%}a{color:#058;text-decoration:none;transition:color .3s ease-in-out}a:hover{color:#e82}.title code{font-weight:bold}h2[id^="header-"]{margin-top:2em}.ident{color:#900}pre code{background:#f8f8f8;font-size:.8em;line-height:1.4em}code{background:#f2f2f1;padding:1px 4px;overflow-wrap:break-word}h1 code{background:transparent}pre{background:#f8f8f8;border:0;border-top:1px solid #ccc;border-bottom:1px solid #ccc;margin:1em 0;padding:1ex}#http-server-module-list{display:flex;flex-flow:column}#http-server-module-list div{display:flex}#http-server-module-list dt{min-width:10%}#http-server-module-list p{margin-top:0}.toc ul,#index{list-style-type:none;margin:0;padding:0}#index code{background:transparent}#index h3{border-bottom:1px solid #ddd}#index ul{padding:0}#index h4{font-weight:bold}#index h4 + ul{margin-bottom:.6em}@media (min-width:200ex){#index .two-column{column-count:2}}@media (min-width:300ex){#index .two-column{column-count:3}}dl{margin-bottom:2em}dl dl:last-child{margin-bottom:4em}dd{margin:0 0 1em 3em}#header-classes + dl > dd{margin-bottom:3em}dd dd{margin-left:2em}dd p{margin:10px 0}.name{background:#eee;font-weight:bold;font-size:.85em;padding:5px 10px;display:inline-block;min-width:40%}.name:hover{background:#e0e0e0}.name > span:first-child{white-space:nowrap}.name.class > span:nth-child(2){margin-left:.4em}.inherited{color:#999;border-left:5px solid #eee;padding-left:1em}.inheritance em{font-style:normal;font-weight:bold}.desc h2{font-weight:400;font-size:1.25em}.desc h3{font-size:1em}.desc dt code{background:inherit}.source summary{color:#666;text-align:right;font-weight:400;font-size:.8em;text-transform:uppercase;cursor:pointer}.source pre{max-height:500px;overflow:auto;margin:0}.source pre code{font-size:12px;overflow:visible}.hlist{list-style:none}.hlist li{display:inline}.hlist li:after{content:',\2002'}.hlist li:last-child:after{content:none}.hlist .hlist{display:inline;padding-left:1em}img{max-width:100%}.admonition{padding:.1em .5em;margin-bottom:1em}.admonition-title{font-weight:bold}.admonition.note,.admonition.info,.admonition.important{background:#aef}.admonition.todo,.admonition.versionadded,.admonition.tip,.admonition.hint{background:#dfd}.admonition.warning,.admonition.versionchanged,.admonition.deprecated{background:#fd4}.admonition.error,.admonition.danger,.admonition.caution{background:lightpink}</style>
+<style media="screen and (min-width: 700px)">@media screen and (min-width:700px){#sidebar{width:30%}#content{width:70%;max-width:100ch;padding:3em 4em;border-left:1px solid #ddd}pre code{font-size:1em}.item .name{font-size:1em}main{display:flex;flex-direction:row-reverse;justify-content:flex-end}.toc ul ul,#index ul{padding-left:1.5em}.toc > ul > li{margin-top:.5em}}</style>
+<style media="print">@media print{#sidebar h1{page-break-before:always}.source{display:none}}@media print{*{background:transparent !important;color:#000 !important;box-shadow:none !important;text-shadow:none !important}a[href]:after{content:" (" attr(href) ")";font-size:90%}a[href][title]:after{content:none}abbr[title]:after{content:" (" attr(title) ")"}.ir a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}@page{margin:0.5cm}p,h2,h3{orphans:3;widows:3}h1,h2,h3,h4,h5,h6{page-break-after:avoid}}</style>
+</head>
+<body>
+<main>
+<article id="content">
+<header>
+<h1 class="title">Package <code>mongiris</code></h1>
+</header>
+<section id="section-intro">
+The package <code>mongiris</code> consists of two modules:
+<ul>
+ <li><a href="integrator.html">integrator</a>, for integrating data sources. There should be no need to run this module since the MongoDB dumps are provided.</li>
+ <li><a href=""api.html>api</a>, for manipulating IRIS data.</li>
+</ul>
+
+</section>
+</article>
+ <nav id="sidebar">
+<h1>Index</h1>
+<div class="toc">
+<ul></ul>
+</div>
+<ul id="index">
+<li><h3><a href="#header-modules">Modules</a></h3>
+<ul>
+<li>
+<h4><code><a title="api" href="api.html">api</a></code></h4>
+</li>
+<li><h4><code><a title="integrator" href="integrator.html">integrator</a></code></h4>
+</li>
+</ul>
+</li>
+</ul>
+</nav>
+</main>
+<footer id="footer">
+<p>Generated by <a href="https://pdoc3.github.io/pdoc"><cite>pdoc</cite> 0.6.3</a>.</p>
+</footer>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+<script>hljs.initHighlightingOnLoad()</script>
+</main>
+</body>
+</html>
\ No newline at end of file
diff --git a/html/integrator.html b/html/integrator.html
new file mode 100644
index 0000000000000000000000000000000000000000..84b0555319be698064ac6c3e078794127eea044a
--- /dev/null
+++ b/html/integrator.html
@@ -0,0 +1,390 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1" />
+<meta name="generator" content="pdoc 0.6.3" />
+<title>integrator API documentation</title>
+<meta name="description" content="" />
+<link href='https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.0/normalize.min.css' rel='stylesheet'>
+<link href='https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/8.0.0/sanitize.min.css' rel='stylesheet'>
+<link href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css" rel="stylesheet">
+<style>.flex{display:flex !important}body{line-height:1.5em}#content{padding:20px}#sidebar{padding:30px;overflow:hidden}.http-server-breadcrumbs{font-size:130%;margin:0 0 15px 0}#footer{font-size:.75em;padding:5px 30px;border-top:1px solid #ddd;text-align:right}#footer p{margin:0 0 0 1em;display:inline-block}#footer p:last-child{margin-right:30px}h1,h2,h3,h4,h5{font-weight:300}h1{font-size:2.5em;line-height:1.1em}h2{font-size:1.75em;margin:1em 0 .50em 0}h3{font-size:1.4em;margin:25px 0 10px 0}h4{margin:0;font-size:105%}a{color:#058;text-decoration:none;transition:color .3s ease-in-out}a:hover{color:#e82}.title code{font-weight:bold}h2[id^="header-"]{margin-top:2em}.ident{color:#900}pre code{background:#f8f8f8;font-size:.8em;line-height:1.4em}code{background:#f2f2f1;padding:1px 4px;overflow-wrap:break-word}h1 code{background:transparent}pre{background:#f8f8f8;border:0;border-top:1px solid #ccc;border-bottom:1px solid #ccc;margin:1em 0;padding:1ex}#http-server-module-list{display:flex;flex-flow:column}#http-server-module-list div{display:flex}#http-server-module-list dt{min-width:10%}#http-server-module-list p{margin-top:0}.toc ul,#index{list-style-type:none;margin:0;padding:0}#index code{background:transparent}#index h3{border-bottom:1px solid #ddd}#index ul{padding:0}#index h4{font-weight:bold}#index h4 + ul{margin-bottom:.6em}@media (min-width:200ex){#index .two-column{column-count:2}}@media (min-width:300ex){#index .two-column{column-count:3}}dl{margin-bottom:2em}dl dl:last-child{margin-bottom:4em}dd{margin:0 0 1em 3em}#header-classes + dl > dd{margin-bottom:3em}dd dd{margin-left:2em}dd p{margin:10px 0}.name{background:#eee;font-weight:bold;font-size:.85em;padding:5px 10px;display:inline-block;min-width:40%}.name:hover{background:#e0e0e0}.name > span:first-child{white-space:nowrap}.name.class > span:nth-child(2){margin-left:.4em}.inherited{color:#999;border-left:5px solid #eee;padding-left:1em}.inheritance em{font-style:normal;font-weight:bold}.desc h2{font-weight:400;font-size:1.25em}.desc h3{font-size:1em}.desc dt code{background:inherit}.source summary{color:#666;text-align:right;font-weight:400;font-size:.8em;text-transform:uppercase;cursor:pointer}.source pre{max-height:500px;overflow:auto;margin:0}.source pre code{font-size:12px;overflow:visible}.hlist{list-style:none}.hlist li{display:inline}.hlist li:after{content:',\2002'}.hlist li:last-child:after{content:none}.hlist .hlist{display:inline;padding-left:1em}img{max-width:100%}.admonition{padding:.1em .5em;margin-bottom:1em}.admonition-title{font-weight:bold}.admonition.note,.admonition.info,.admonition.important{background:#aef}.admonition.todo,.admonition.versionadded,.admonition.tip,.admonition.hint{background:#dfd}.admonition.warning,.admonition.versionchanged,.admonition.deprecated{background:#fd4}.admonition.error,.admonition.danger,.admonition.caution{background:lightpink}</style>
+<style media="screen and (min-width: 700px)">@media screen and (min-width:700px){#sidebar{width:30%}#content{width:70%;max-width:100ch;padding:3em 4em;border-left:1px solid #ddd}pre code{font-size:1em}.item .name{font-size:1em}main{display:flex;flex-direction:row-reverse;justify-content:flex-end}.toc ul ul,#index ul{padding-left:1.5em}.toc > ul > li{margin-top:.5em}}</style>
+<style media="print">@media print{#sidebar h1{page-break-before:always}.source{display:none}}@media print{*{background:transparent !important;color:#000 !important;box-shadow:none !important;text-shadow:none !important}a[href]:after{content:" (" attr(href) ")";font-size:90%}a[href][title]:after{content:none}abbr[title]:after{content:" (" attr(title) ")"}.ir a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}@page{margin:0.5cm}p,h2,h3{orphans:3;widows:3}h1,h2,h3,h4,h5,h6{page-break-after:avoid}}</style>
+</head>
+<body>
+<main>
+<article id="content">
+<header>
+<h1 class="title">Module <code>integrator</code></h1>
+</header>
+<section id="section-intro">
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">#!/usr/bin/env python
+# encoding: utf-8
+# =============================================================================
+# Integrator: perform the integration of different data sources to produce MongoDB documents (GeoJSON format) containing
+# IRIS information (~640 or ~350 INSEE indicators, depending on the IRIS).
+# This file does not need to be run because the MongoDB dump is provided (unless for integrating new data sources).
+# =============================================================================
+# Path to MongoDB tools (under MacOS): /Applications/MongoDB.app/Contents/Resources/Vendor/mongodb/bin/
+# Export and import a local database (metadata/indexes included, option dryRun for testing, several files):
+# mongodump --db dbinsee --verbose -o dump/
+# mongorestore --db dbinsee --dir dump/dbinsee --verbose --dryRun
+# Export and import a local database (metadata/indexes included, option dryRun for testing, single binary file):
+# mongodump --db dbinsee --verbose --archive=dbinsee.bin
+# mongorestore --archive=dbinsee.bin --verbose
+# Export and import in JSON:
+# ./mongoexport --db dbinsee --out ~/dump-iris.json
+# ./mongoimport --db dbinsee --file ~/dump-iris.json
+# =============================================================================
+
+import os
+import logging
+from mongiris import config
+from mongiris import xls_utils
+from mongiris import api
+
+
+def get_all_xlsx_files(dir_sources):
+ """
+ Returns a list of spreadsheets files to be integrated.
+
+ Args:
+ dir_sources: path to a directory containing data sources (spreadsheets from INSEE)
+
+ Returns:
+ insee_files: a list of filepaths (to xls files)
+
+ """
+ insee_files = list()
+ for file in os.listdir(dir_sources):
+ filepath = os.path.join(config.insee_dir, file)
+ if os.path.isfile(filepath) and filepath.endswith(".xls"):
+ insee_files.append(filepath)
+ return insee_files
+
+
+def integrate_xls_file(connection, xls_file):
+ """
+ Integrates data from an xsl file : the 3 collections may be updated. First, extracts information about the source
+ (e.g., title, date) and inserts a document for that source in 'collsources'. Next extracts information about the
+ indicators (e.g., short name, long name) and updates/inserts a document in 'collindic'. Finally for each line (IRIS)
+ in the spreadsheet, updates corresponding document with new indicators.
+
+ Args:
+ connection: an object representing the database connection
+ xls_file: a csv file path containing INSEE indicators
+ """
+ indicator_metadata, indicators, source_metadata = xls_utils.parse_xls_to_dict(xls_file)
+
+ # add the source metadata into collection_sources
+ doc = connection.find_one_document(connection.collection_sources, {"filename": xls_file})
+ if doc is None:
+ connection.insert_one_document(connection.collection_sources, source_metadata)
+
+ # add the indicators information into collection_indic
+ for ind in indicator_metadata:
+ short_name = ind[config.geojson_shortname_label]
+ doc = connection.find_one_document(connection.collection_indic, {config.geojson_shortname_label: short_name})
+ if doc is not None: # only update field from_insee_files, $addToSet does not add duplicate values
+ connection.update_one_document(connection.collection_indic, {config.geojson_shortname_label: short_name},
+ {"$addToSet": {config.geojson_from_files_label: xls_file}})
+ else: # add the document
+ connection.insert_one_document(connection.collection_indic, ind)
+
+ # add the indicators values into collection_iris
+ not_found_iris = found_iris = nb_replacements = 0
+ for code_iris, indics in indicators.items():
+ doc = connection.get_iris_from_code(code_iris)
+ if doc is None:
+ not_found_iris += 1
+ else:
+ doc_id = doc['properties']['CODE_IRIS']
+ found_iris += 1
+ need_replacement = False
+ doc_changes = {"$set": dict()} # an update document containing the new fields
+ for ind, value in indics.items():
+ if ind in config.labels_dictionary_init: # main indicator, do not replace it if already there
+ if ind not in doc['properties']: # general information (code iris, city name, etc.)
+ doc_changes["$set"]["properties." + ind] = value
+ need_replacement = True
+ else: # raw indicator
+ if ind not in doc['properties'][config.geojson_raw_indicators_label]: # add this value
+ doc_changes["$set"]["properties." + config.geojson_raw_indicators_label + "." + ind] = value
+ need_replacement = True
+
+ if need_replacement: # replace the old doc by new doc
+ res = connection.update_one_document(connection.collection_iris, {'properties.CODE_IRIS': doc_id}, doc_changes)
+ nb_replacements += res.modified_count
+
+ logger.info(f"\t\t{xls_file}: {found_iris} found iris, {not_found_iris} not found iris, {nb_replacements} updates")
+ return True
+
+
+def integrate(connection, dir_sources):
+ """
+ Main integration method: parses a directory of data sources, and call the `integrate_xls_file` function for each
+ data source.
+
+ Args:
+ connection: an object representing the database connection
+ dir_sources: path to a directory containing data sources (spreadsheets from INSEE)
+ """
+ logger.info("Searching xlsx files...")
+ insee_files = get_all_xlsx_files(dir_sources) # get the list of all xlsx files to be integrated
+ logger.info(f"Found a total of {len(insee_files)} xlsx files to be integrated.")
+ # insee_files = [] # roughly 10-20 mins execution time for one data source
+ logger.info(f'Selected {len(insee_files)} xlsx files to be integrated.')
+
+ logger.info("Integrating sources files (metadata for source and indicators, data for iris)")
+ for file in insee_files: # integrate each xlsx INSEE file (both indicators in IRIS and indicators dict)
+ logger.info(f"\t- INSEE xlsx file: {file}")
+ integrate_xls_file(connection, file)
+ logger.info("Done !")
+
+
+def check_properties(connection):
+ """
+ Computes and prints statistics on the IRIS collection, mainly the number of indicators per document.
+ Args:
+ connection: an object representing the database connection
+ """
+ docs = connection.find_documents(connection.collection_iris, {}, )
+ counts = dict()
+ for doc in docs:
+ nb = len(doc['properties'][config.geojson_raw_indicators_label])
+ if nb not in counts:
+ counts[nb] = 0
+ counts[nb] += 1
+ print(counts)
+
+
+#########################
+# main integration script
+#########################
+
+if __name__ == '__main__':
+
+ logging.basicConfig(format='[%(levelname)s] - %(name)s - %(asctime)s : %(message)s')
+ logger = logging.getLogger()
+ logger.setLevel(logging.INFO)
+
+ connection = api.Mongiris() # database connection
+ integrate(connection, config.insee_dir) # integrating data sources
+ check_properties(connection) # stats about properties 5min execution {350: 36530, 638: 11738, 615: 1057, 373: 79}</code></pre>
+</details>
+</section>
+<section>
+</section>
+<section>
+</section>
+<section>
+<h2 class="section-title" id="header-functions">Functions</h2>
+<dl>
+<dt id="integrator.check_properties"><code class="name flex">
+<span>def <span class="ident">check_properties</span></span>(<span>connection)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Computes and prints statistics on the IRIS collection, mainly the number of indicators per document.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>connection</code></strong></dt>
+<dd>an object representing the database connection</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def check_properties(connection):
+ """
+ Computes and prints statistics on the IRIS collection, mainly the number of indicators per document.
+ Args:
+ connection: an object representing the database connection
+ """
+ docs = connection.find_documents(connection.collection_iris, {}, )
+ counts = dict()
+ for doc in docs:
+ nb = len(doc['properties'][config.geojson_raw_indicators_label])
+ if nb not in counts:
+ counts[nb] = 0
+ counts[nb] += 1
+ print(counts)</code></pre>
+</details>
+</dd>
+<dt id="integrator.get_all_xlsx_files"><code class="name flex">
+<span>def <span class="ident">get_all_xlsx_files</span></span>(<span>dir_sources)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Returns a list of spreadsheets files to be integrated.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>dir_sources</code></strong></dt>
+<dd>path to a directory containing data sources (spreadsheets from INSEE)</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>insee_files</code></strong></dt>
+<dd>a list of filepaths (to xls files)</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def get_all_xlsx_files(dir_sources):
+ """
+ Returns a list of spreadsheets files to be integrated.
+
+ Args:
+ dir_sources: path to a directory containing data sources (spreadsheets from INSEE)
+
+ Returns:
+ insee_files: a list of filepaths (to xls files)
+
+ """
+ insee_files = list()
+ for file in os.listdir(dir_sources):
+ filepath = os.path.join(config.insee_dir, file)
+ if os.path.isfile(filepath) and filepath.endswith(".xls"):
+ insee_files.append(filepath)
+ return insee_files</code></pre>
+</details>
+</dd>
+<dt id="integrator.integrate"><code class="name flex">
+<span>def <span class="ident">integrate</span></span>(<span>connection, dir_sources)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Main integration method: parses a directory of data sources, and call the <a title="integrator.integrate_xls_file" href="#integrator.integrate_xls_file"><code>integrate_xls_file()</code></a> function for each
+data source.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>connection</code></strong></dt>
+<dd>an object representing the database connection</dd>
+<dt><strong><code>dir_sources</code></strong></dt>
+<dd>path to a directory containing data sources (spreadsheets from INSEE)</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def integrate(connection, dir_sources):
+ """
+ Main integration method: parses a directory of data sources, and call the `integrate_xls_file` function for each
+ data source.
+
+ Args:
+ connection: an object representing the database connection
+ dir_sources: path to a directory containing data sources (spreadsheets from INSEE)
+ """
+ logger.info("Searching xlsx files...")
+ insee_files = get_all_xlsx_files(dir_sources) # get the list of all xlsx files to be integrated
+ logger.info(f"Found a total of {len(insee_files)} xlsx files to be integrated.")
+ # insee_files = [] # roughly 10-20 mins execution time for one data source
+ logger.info(f'Selected {len(insee_files)} xlsx files to be integrated.')
+
+ logger.info("Integrating sources files (metadata for source and indicators, data for iris)")
+ for file in insee_files: # integrate each xlsx INSEE file (both indicators in IRIS and indicators dict)
+ logger.info(f"\t- INSEE xlsx file: {file}")
+ integrate_xls_file(connection, file)
+ logger.info("Done !")</code></pre>
+</details>
+</dd>
+<dt id="integrator.integrate_xls_file"><code class="name flex">
+<span>def <span class="ident">integrate_xls_file</span></span>(<span>connection, xls_file)</span>
+</code></dt>
+<dd>
+<section class="desc"><p>Integrates data from an xsl file : the 3 collections may be updated. First, extracts information about the source
+(e.g., title, date) and inserts a document for that source in 'collsources'. Next extracts information about the
+indicators (e.g., short name, long name) and updates/inserts a document in 'collindic'. Finally for each line (IRIS)
+in the spreadsheet, updates corresponding document with new indicators.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>connection</code></strong></dt>
+<dd>an object representing the database connection</dd>
+<dt><strong><code>xls_file</code></strong></dt>
+<dd>a csv file path containing INSEE indicators</dd>
+</dl></section>
+<details class="source">
+<summary>Source code</summary>
+<pre><code class="python">def integrate_xls_file(connection, xls_file):
+ """
+ Integrates data from an xsl file : the 3 collections may be updated. First, extracts information about the source
+ (e.g., title, date) and inserts a document for that source in 'collsources'. Next extracts information about the
+ indicators (e.g., short name, long name) and updates/inserts a document in 'collindic'. Finally for each line (IRIS)
+ in the spreadsheet, updates corresponding document with new indicators.
+
+ Args:
+ connection: an object representing the database connection
+ xls_file: a csv file path containing INSEE indicators
+ """
+ indicator_metadata, indicators, source_metadata = xls_utils.parse_xls_to_dict(xls_file)
+
+ # add the source metadata into collection_sources
+ doc = connection.find_one_document(connection.collection_sources, {"filename": xls_file})
+ if doc is None:
+ connection.insert_one_document(connection.collection_sources, source_metadata)
+
+ # add the indicators information into collection_indic
+ for ind in indicator_metadata:
+ short_name = ind[config.geojson_shortname_label]
+ doc = connection.find_one_document(connection.collection_indic, {config.geojson_shortname_label: short_name})
+ if doc is not None: # only update field from_insee_files, $addToSet does not add duplicate values
+ connection.update_one_document(connection.collection_indic, {config.geojson_shortname_label: short_name},
+ {"$addToSet": {config.geojson_from_files_label: xls_file}})
+ else: # add the document
+ connection.insert_one_document(connection.collection_indic, ind)
+
+ # add the indicators values into collection_iris
+ not_found_iris = found_iris = nb_replacements = 0
+ for code_iris, indics in indicators.items():
+ doc = connection.get_iris_from_code(code_iris)
+ if doc is None:
+ not_found_iris += 1
+ else:
+ doc_id = doc['properties']['CODE_IRIS']
+ found_iris += 1
+ need_replacement = False
+ doc_changes = {"$set": dict()} # an update document containing the new fields
+ for ind, value in indics.items():
+ if ind in config.labels_dictionary_init: # main indicator, do not replace it if already there
+ if ind not in doc['properties']: # general information (code iris, city name, etc.)
+ doc_changes["$set"]["properties." + ind] = value
+ need_replacement = True
+ else: # raw indicator
+ if ind not in doc['properties'][config.geojson_raw_indicators_label]: # add this value
+ doc_changes["$set"]["properties." + config.geojson_raw_indicators_label + "." + ind] = value
+ need_replacement = True
+
+ if need_replacement: # replace the old doc by new doc
+ res = connection.update_one_document(connection.collection_iris, {'properties.CODE_IRIS': doc_id}, doc_changes)
+ nb_replacements += res.modified_count
+
+ logger.info(f"\t\t{xls_file}: {found_iris} found iris, {not_found_iris} not found iris, {nb_replacements} updates")
+ return True</code></pre>
+</details>
+</dd>
+</dl>
+</section>
+<section>
+</section>
+</article>
+<nav id="sidebar">
+<h1>Index</h1>
+<div class="toc">
+<ul></ul>
+</div>
+<ul id="index">
+<li><h3><a href="#header-functions">Functions</a></h3>
+<ul class="">
+<li><code><a title="integrator.check_properties" href="#integrator.check_properties">check_properties</a></code></li>
+<li><code><a title="integrator.get_all_xlsx_files" href="#integrator.get_all_xlsx_files">get_all_xlsx_files</a></code></li>
+<li><code><a title="integrator.integrate" href="#integrator.integrate">integrate</a></code></li>
+<li><code><a title="integrator.integrate_xls_file" href="#integrator.integrate_xls_file">integrate_xls_file</a></code></li>
+</ul>
+</li>
+</ul>
+</nav>
+</main>
+<footer id="footer">
+<p>Generated by <a href="https://pdoc3.github.io/pdoc"><cite>pdoc</cite> 0.6.3</a>.</p>
+</footer>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+<script>hljs.initHighlightingOnLoad()</script>
+</body>
+</html>
\ No newline at end of file
diff --git a/mongiris/__init__.py b/mongiris/__init__.py
index f3e41e8d824d2ca3f37b6035cf3ee4dfb181a1bb..3b2918629b05efcf4a3869f258045f46f25a8e1c 100644
--- a/mongiris/__init__.py
+++ b/mongiris/__init__.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# encoding: utf-8
-from .main import Mongiris
+from .api import Mongiris
# all modules
__all__ = ['Mongiris', ]
diff --git a/mongiris/main.py b/mongiris/api.py
similarity index 98%
rename from mongiris/main.py
rename to mongiris/api.py
index 6c3c56a26e71e337ef093fb3502935f6914656c1..6eb822bfe667e6e33591256d51c24e2bedff5fc8 100755
--- a/mongiris/main.py
+++ b/mongiris/api.py
@@ -15,7 +15,7 @@ from mongiris import config
class Mongiris:
"""
- The Mongiris class is an API to manipulate data from the MongoDB 'dbinsee' datatase.
+ The Mongiris class is an API to manipulate data from the MongoDB `dbinsee` datatase.
Several methods accepts a 'collection' parameter for flexibility (i.e., be able to query other collections than the
iris collection).
@@ -25,17 +25,19 @@ class Mongiris:
The constructor initializes the logger and it automatically connects to the database.
The name of the database and of the three collections
are based on the default names (the ones from the dump). If the names are changed in MongoDB, they should be changed
- in the `mongiris/config.py` file.
+ in the `config.py` file.
- Examples of usages, including testing geospatial queries, are available in the `mongiris.tests.mongiris_test.py`
- file.
+ Examples of usages, including testing geospatial queries, are available in `tests/api_tests.py`.
- An example of IRIS following the GeoJSON format is provided in `mongiris.data.example-iris.json`.
+ An example of IRIS following the GeoJSON format is provided in `data/example-iris.json`.
Additional resources:
+
- [MongoDB documentation](http://www.mongodb.org/)
+
- [GeoJSON format specifications](https://geojson.org/)
- - [pymongo](http://api.mongodb.com/python/current/)
+
+ - [pymongo API](http://api.mongodb.com/python/current/) between Python and MongoDB
"""
diff --git a/mongiris/integrator.py b/mongiris/integrator.py
index e2d8d7e11ea4569690785cfa59c61d32e323782d..78ed3a7118729ce9ffe296ea61ff0339c972f31c 100644
--- a/mongiris/integrator.py
+++ b/mongiris/integrator.py
@@ -2,7 +2,7 @@
# encoding: utf-8
# =============================================================================
# Integrator: perform the integration of different data sources to produce MongoDB documents (GeoJSON format) containing
-# IRIS information (~640 INSEE indicators).
+# IRIS information (~640 or ~350 INSEE indicators, depending on the IRIS).
# This file does not need to be run because the MongoDB dump is provided (unless for integrating new data sources).
# =============================================================================
# Path to MongoDB tools (under MacOS): /Applications/MongoDB.app/Contents/Resources/Vendor/mongodb/bin/
@@ -21,46 +21,60 @@ import os
import logging
from mongiris import config
from mongiris import xls_utils
-from mongiris import main
+from mongiris import api
-def get_all_xlsx_files(path_dir_xlsx):
- # generate a list of INSEE xlsx files contained in directory path_dir_xlsx
+def get_all_xlsx_files(dir_sources):
+ """
+ Returns a list of spreadsheets files to be integrated.
+
+ Args:
+ dir_sources: path to a directory containing data sources (spreadsheets from INSEE)
+
+ Returns:
+ insee_files: a list of filepaths (to xls files)
+
+ """
insee_files = list()
- for file in os.listdir(path_dir_xlsx):
+ for file in os.listdir(dir_sources):
filepath = os.path.join(config.insee_dir, file)
if os.path.isfile(filepath) and filepath.endswith(".xls"):
insee_files.append(filepath)
return insee_files
-def integrate_xls_file(xls_file):
+def integrate_xls_file(connection, xls_file):
"""
- Integrate data from the xsl file to update an IRIS and its indicators.
- :param xls_file: a csv file path containing INSEE indicators
- :return: nothing ;)
+ Integrates data from an xsl file : the 3 collections may be updated. First, extracts information about the source
+ (e.g., title, date) and inserts a document for that source in 'collsources'. Next extracts information about the
+ indicators (e.g., short name, long name) and updates/inserts a document in 'collindic'. Finally for each line (IRIS)
+ in the spreadsheet, updates corresponding document with new indicators.
+
+ Args:
+ connection: an object representing the database connection
+ xls_file: a csv file path containing INSEE indicators
"""
indicator_metadata, indicators, source_metadata = xls_utils.parse_xls_to_dict(xls_file)
# add the source metadata into collection_sources
- doc = connexion.find_one_document(connexion.collection_sources, {"filename": xls_file})
+ doc = connection.find_one_document(connection.collection_sources, {"filename": xls_file})
if doc is None:
- connexion.insert_one_document(connexion.collection_sources, source_metadata)
+ connection.insert_one_document(connection.collection_sources, source_metadata)
# add the indicators information into collection_indic
for ind in indicator_metadata:
short_name = ind[config.geojson_shortname_label]
- doc = connexion.find_one_document(connexion.collection_indic, {config.geojson_shortname_label: short_name})
+ doc = connection.find_one_document(connection.collection_indic, {config.geojson_shortname_label: short_name})
if doc is not None: # only update field from_insee_files, $addToSet does not add duplicate values
- connexion.update_one_document(connexion.collection_indic, {config.geojson_shortname_label: short_name},
+ connection.update_one_document(connection.collection_indic, {config.geojson_shortname_label: short_name},
{"$addToSet": {config.geojson_from_files_label: xls_file}})
else: # add the document
- connexion.insert_one_document(connexion.collection_indic, ind)
+ connection.insert_one_document(connection.collection_indic, ind)
# add the indicators values into collection_iris
not_found_iris = found_iris = nb_replacements = 0
for code_iris, indics in indicators.items():
- doc = connexion.get_iris_from_code(code_iris)
+ doc = connection.get_iris_from_code(code_iris)
if doc is None:
not_found_iris += 1
else:
@@ -79,17 +93,42 @@ def integrate_xls_file(xls_file):
need_replacement = True
if need_replacement: # replace the old doc by new doc
- res = connexion.update_one_document(connexion.collection_iris, {'properties.CODE_IRIS': doc_id}, doc_changes)
+ res = connection.update_one_document(connection.collection_iris, {'properties.CODE_IRIS': doc_id}, doc_changes)
nb_replacements += res.modified_count
logger.info(f"\t\t{xls_file}: {found_iris} found iris, {not_found_iris} not found iris, {nb_replacements} updates")
-
return True
-def check_properties():
- # count the number of indicators for each document/iris and prints statistics
- docs = connexion.find_documents(connexion.collection_iris, {}, )
+def integrate(connection, dir_sources):
+ """
+ Main integration method: parses a directory of data sources, and call the `integrate_xls_file` function for each
+ data source.
+
+ Args:
+ connection: an object representing the database connection
+ dir_sources: path to a directory containing data sources (spreadsheets from INSEE)
+ """
+ logger.info("Searching xlsx files...")
+ insee_files = get_all_xlsx_files(dir_sources) # get the list of all xlsx files to be integrated
+ logger.info(f"Found a total of {len(insee_files)} xlsx files to be integrated.")
+ # insee_files = [] # roughly 10-20 mins execution time for one data source
+ logger.info(f'Selected {len(insee_files)} xlsx files to be integrated.')
+
+ logger.info("Integrating sources files (metadata for source and indicators, data for iris)")
+ for file in insee_files: # integrate each xlsx INSEE file (both indicators in IRIS and indicators dict)
+ logger.info(f"\t- INSEE xlsx file: {file}")
+ integrate_xls_file(connection, file)
+ logger.info("Done !")
+
+
+def check_properties(connection):
+ """
+ Computes and prints statistics on the IRIS collection, mainly the number of indicators per document.
+ Args:
+ connection: an object representing the database connection
+ """
+ docs = connection.find_documents(connection.collection_iris, {}, )
counts = dict()
for doc in docs:
nb = len(doc['properties'][config.geojson_raw_indicators_label])
@@ -109,22 +148,7 @@ if __name__ == '__main__':
logger = logging.getLogger()
logger.setLevel(logging.INFO)
- connexion = main.Mongiris() # database connection
-
- logger.info("Searching xlsx files...")
- insee_files = get_all_xlsx_files(config.insee_dir) # get the list of all xlsx files to be integrated
- logger.info(f"Found a total of {len(insee_files)} xlsx files to be integrated.")
- #insee_files = [] # roughly 10-20 mins execution time for one source
- logger.info(f'Selected {len(insee_files)} xlsx files to be integrated.')
-
- logger.info("Initializing dictionary files")
- dict_labels = dict(config.labels_dictionary_init) # the initial dictionary (labels of main indicators)
-
- logger.info("Integrating sources files (metadata for source and indicators, data for iris)")
- for file in insee_files: # integrate each xlsx INSEE file (both indicators in IRIS and indicators dict)
- logger.info(f"\t- INSEE xlsx file: {file}")
- integrate_xls_file(file)
-
- check_properties() # 5min execution {350: 36530, 638: 11738, 615: 1057, 373: 79}
+ connection = api.Mongiris() # database connection
+ integrate(connection, config.insee_dir) # integrating data sources
+ check_properties(connection) # stats about properties 5min execution {350: 36530, 638: 11738, 615: 1057, 373: 79}
- logger.info("Done !")
diff --git a/mongiris/tests/mongiris_tests.py b/mongiris/tests/api_tests.py
similarity index 99%
rename from mongiris/tests/mongiris_tests.py
rename to mongiris/tests/api_tests.py
index d31199aeaa641c10d4f949243dbbf246a585a659..77ddba1d4cb9c611fed94c1dc32533e7910ec166 100644
--- a/mongiris/tests/mongiris_tests.py
+++ b/mongiris/tests/api_tests.py
@@ -4,7 +4,7 @@
# Unit tests for mongiris.
# =============================================================================
-from mongiris.main import Mongiris
+from mongiris.api import Mongiris
import unittest
import random
import re
diff --git a/mongiris/tests/dummy.py b/mongiris/tests/dummy.py
index 32bedfa29ec13daef461aed6a8e2d256c0531c87..6e4a216b91f65dabae73b6156702c476ebee4f6b 100644
--- a/mongiris/tests/dummy.py
+++ b/mongiris/tests/dummy.py
@@ -1,5 +1,5 @@
-from mongiris.main import Mongiris
+from mongiris.api import Mongiris
import json
diff --git a/paper.md b/paper.md
index d0aff7f9c518a89bdc6be147b0f69e8a4ad0d78c..8b765a7f642c8635542251b700e82684c6ee8f43 100644
--- a/paper.md
+++ b/paper.md
@@ -33,6 +33,8 @@ For these reasons, we propose the package Mongiris, which includes integrated da
The package is composed of two modules: integration and API.
+The integration module is responsible for extracting information from data sources
+
discuss the integration (evolution of data) -> need to comment integrator + simplify the __main__
dire que integration = juste pour evolution , mais dum deja fourni - cf stats sur les indicateurs :
{350: 36530, 638: 11738, 615: 1057, 373: 79}