fix: Smart/Origin: main hook in fill_index.py

c2corg_api/__init__.py

@@ -226,7 +226,7 @@ def process_new_waypoint(mapper, connection, geometry):
         stop_id = place["id"]
 
         # Get informations of stopareas to know its transports
-        stop_info_url = "https://api.navitia.io/v1/places/%d", stop_id
+        stop_info_url = f"https://api.navitia.io/v1/places/{stop_id}"
         stop_info_response = requests.get(
             stop_info_url, headers=navitia_headers
         )
@@ -619,8 +619,12 @@ def _update_route_duration(connection, route_id, calculated_duration_in_days):
         ),
         {"duration": calculated_duration_in_days, "route_id": route_id},
     )
+    if (calculated_duration_in_days is None):
+        calculated_duration_in_days_str = "None"
+    else:
+        calculated_duration_in_days_str = "%f", calculated_duration_in_days
     log.info(
-        "Route %d: Database updated with calculated_duration = %f days.",
+        "Route %d: Database updated with calculated_duration = %s days.",
         route_id,
-        calculated_duration_in_days
+        calculated_duration_in_days_str
     )

c2corg_api/scripts/es/create_index.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""
+Creates (or recreates) the ES index with the correct mappings.
+Must be run before fill_index.py when building the index from scratch.
+
+Usage: python create_index.py <config>.ini
+"""
+import sys
+import logging
+
+from pyramid.paster import get_appsettings, setup_logging
+from pyramid.scripts.common import parse_vars
+
+from c2corg_api.search import configure_es_from_config, elasticsearch_config, \
+    search_documents
+from c2corg_api.search.mapping import analysis_settings
+
+
+def main(argv=sys.argv):
+    if len(argv) < 2:
+        print('Usage: %s <config_uri> [var=value]' % argv[0])
+        sys.exit(1)
+
+    config_uri = argv[1]
+    options = parse_vars(argv[2:])
+    setup_logging(config_uri)
+    logging.getLogger('sqlalchemy.engine').setLevel(logging.ERROR)
+
+    settings = get_appsettings(config_uri, options=options)
+    configure_es_from_config(settings)
+
+    client = elasticsearch_config['client']
+    index_name = elasticsearch_config['index']
+
+    # Delete index if it exists (to fix potentially wrong mappings)
+    if client.indices.exists(index=index_name):
+        print('Deleting existing index: {}'.format(index_name))
+        client.indices.delete(index=index_name)
+
+    # Create index with analysis settings
+    print('Creating index: {}'.format(index_name))
+    client.indices.create(index=index_name, body={
+        'settings': {
+            'analysis': analysis_settings
+        }
+    })
+
+    # Apply the correct mapping for every document type
+    # (if we let ES infer types by directly running fill_index.py,
+    # we could run into type errors on some fields like
+    # a mapping geom -> double instead of geo_point).
+    for doc_type, search_class in search_documents.items():
+        print('Applying mapping for doc_type: {}'.format(doc_type))
+        search_class._doc_type.mapping.save(index_name)
+
+    print('Done. Index "{}" created with correct mappings.'.format(index_name))
+    print('You can now run fill_index.py safely.')
+
+
+if __name__ == '__main__':
+    main()

c2corg_api/scripts/es/fill_index.py

@@ -90,3 +90,6 @@ def progress(count, total_count):
 
     duration = datetime.now() - status['start_time']
     print('Done (duration: {0})'.format(duration))
+
+if __name__ == '__main__':
+    main()

c2corg_api/search/advanced_search.py

@@ -1,10 +1,6 @@
 from c2corg_api.search.mapping_types import meta_param_keys
 from c2corg_api.search.search_filters import build_query
 
-# the chunk size for each elastic search request
-# should be <= ES_MAX_RESULT_WINDOW
-CHUNK_SIZE = 10000
-
 
 def get_search_documents(url_params, meta_params, doc_type):
     """Returns a function that when called with a base-query returns all
@@ -89,13 +85,14 @@ def get_all_filtered_docs(
 
 
 def chunk_ids(ids_set):
+    from c2corg_api.views.document import ES_MAX_RESULT_WINDOW
     """
     Yield successive chunks of IDs from a set/list.
     chunk size is ES_MAX_RESULT_WINDOW
     """
     ids_list = list(ids_set)
-    for i in range(0, len(ids_list), CHUNK_SIZE):
-        yield ids_list[i:i + CHUNK_SIZE]
+    for i in range(0, len(ids_list), ES_MAX_RESULT_WINDOW):
+        yield ids_list[i:i + ES_MAX_RESULT_WINDOW]
 
 
 def contains_search_params(url_params):

c2corg_api/views/navitia.py

@@ -690,6 +690,9 @@ def extract_isochrone_params(request):
             ),
             MIN_TRIP_DURATION * 60
         )
+    else:
+        # default is max trip duration
+        params['boundary_duration[]'] = MAX_TRIP_DURATION * 60
     return params
 
 
@@ -766,8 +769,9 @@ def handle_navitia_response(response):
             response.json().get("error", {}))
 
     elif not response.ok:
-        raise HTTPInternalServerError("Navitia API error: %d ",
-                                      response.status_code)
+        raise HTTPInternalServerError(
+            "Navitia API error: %d " % response.status_code
+        )
 
     return response.json()
 

config/env.default

@@ -31,7 +31,7 @@ redis_cache_key_prefix="c2corg_${instanceid}"
 redis_cache_status_refresh_period=30
 
 image_backend_url="http://images.demov6.camptocamp.org"
-image_url="https://sos.exo.io/c2corg_demov6_active/"
+image_url="https://media.camptocamp.org/c2corg-active"
 
 discourse_url="https://forum.demov6.camptocamp.org"
 discourse_public_url="https://forum.demov6.camptocamp.org"

docker/Dockerfile

@@ -38,4 +38,4 @@ RUN chown -R root:root /var/www \
 EXPOSE 8080
 ENTRYPOINT ["./docker/scripts/entrypoint.sh"]
 
-CMD ["gunicorn", "--paste", "production.ini", "-u", "www-data", "-g", "www-data", "-b", ":8080"]
+CMD ["gunicorn", "--paste", "production.ini", "-u", "www-data", "-g", "www-data", "-b", ":8080", "--timeout", "120"]