fix: test: Smart/Origin unit tests and various fixes

.env

@@ -1,5 +1,5 @@
 # Navitia API Key : 400.000 queries each month
-NAVITIA_API_KEY="" 
+NAVITIA_API_KEY="test-key"
 
 # Maximum distance between a waypoint and a stop, in meters
 MAX_DISTANCE_WAYPOINT_TO_STOPAREA=5000
@@ -8,4 +8,4 @@ MAX_DISTANCE_WAYPOINT_TO_STOPAREA=5000
 WALKING_SPEED=1.12
 
 # Maximum number of stops associated with a waypoint
-MAX_STOP_AREA_FOR_1_WAYPOINT=5
+MAX_STOP_AREA_FOR_1_WAYPOINT=5

.github/workflows/ci.yml

@@ -82,8 +82,16 @@ jobs:
           /tmp/elasticsearch/bin/elasticsearch --daemonize --path.data /tmp
           sleep 30 # ElasticSearch takes few seconds to start, make sure it is available when the build script runs
       - run: curl -v http://localhost:9200
-      - name: Run tests
-        run: export $(cat .env | grep -v "^#" | xargs) && pytest --cov-report term --cov-report xml --cov=c2corg_api
+      - name: Run tests (with secrets)
+        # Accept running on pull requests except if from dependabot
+        if: ${{ github.event_name != 'pull_request' || github.actor != 'dependabot[bot]' }}
+        env:
+          NAVITIA_API_KEY: ${{ secrets.NAVITIA_API_KEY }}
+        run: export $(cat .env | grep -v "^#" | grep -v "^NAVITIA_API_KEY" | xargs) && pytest --cov-report term --cov-report xml --cov=c2corg_api
+      - name: Run tests (without secrets)
+        # secrets are not available for PR from forks, and dependabot PRs
+        if: ${{ github.event_name == 'pull_request' && github.actor == 'dependabot[bot]' }}
+        run: export $(cat .env | grep -v "^#" | xargs) && pytest --cov-report term --cov-report xml --cov=c2corg_api -k "not test_navitia"
       - name: Send coverage to codacy
         # secrets are not available for PR from forks, and dependabot PRs
         if: ${{ github.event_name != 'pull_request' && github.actor != 'dependabot[bot]' }}

c2corg_api/__init__.py

@@ -123,6 +123,14 @@ def delete_waypoint_stopareas(connection, waypoint_id):
 def process_new_waypoint(mapper, connection, geometry):
     """Processes a new waypoint to find its public transports after
     inserting it into documents_geometries."""
+
+    # When GHCI runs without access to GH secrets
+    # (eg. for PR triggered by dependabot)
+    # return immediatly by matching test key from .env
+    api_key = os.getenv("NAVITIA_API_KEY")
+    if api_key == "test-key":
+        return
+
     # Check if document is a waypoint
     waypoint_id = geometry.document_id
 
@@ -147,7 +155,6 @@ def process_new_waypoint(mapper, connection, geometry):
     max_stop_area_for_1_waypoint = int(
         os.getenv("MAX_STOP_AREA_FOR_1_WAYPOINT")
     )
-    api_key = os.getenv("NAVITIA_API_KEY")
     max_duration = int(max_distance_waypoint_to_stoparea / walking_speed)
 
     # increase number of retrieved stop areas,
@@ -226,7 +233,7 @@ def process_new_waypoint(mapper, connection, geometry):
         stop_id = place["id"]
 
         # Get informations of stopareas to know its transports
-        stop_info_url = "https://api.navitia.io/v1/places/%d", stop_id
+        stop_info_url = f"https://api.navitia.io/v1/places/{stop_id}"
         stop_info_response = requests.get(
             stop_info_url, headers=navitia_headers
         )
@@ -619,8 +626,12 @@ def _update_route_duration(connection, route_id, calculated_duration_in_days):
         ),
         {"duration": calculated_duration_in_days, "route_id": route_id},
     )
+    if (calculated_duration_in_days is None):
+        calculated_duration_in_days_str = "None"
+    else:
+        calculated_duration_in_days_str = "%f", calculated_duration_in_days
     log.info(
-        "Route %d: Database updated with calculated_duration = %f days.",
+        "Route %d: Database updated with calculated_duration = %s days.",
         route_id,
-        calculated_duration_in_days
+        calculated_duration_in_days_str
     )

c2corg_api/scripts/es/create_index.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""
+Creates (or recreates) the ES index with the correct mappings.
+Must be run before fill_index.py when building the index from scratch.
+
+Usage: python create_index.py <config>.ini
+"""
+import sys
+import logging
+
+from pyramid.paster import get_appsettings, setup_logging
+from pyramid.scripts.common import parse_vars
+
+from c2corg_api.search import configure_es_from_config, elasticsearch_config, \
+    search_documents
+from c2corg_api.search.mapping import analysis_settings
+
+
+def main(argv=sys.argv):
+    if len(argv) < 2:
+        print('Usage: %s <config_uri> [var=value]' % argv[0])
+        sys.exit(1)
+
+    config_uri = argv[1]
+    options = parse_vars(argv[2:])
+    setup_logging(config_uri)
+    logging.getLogger('sqlalchemy.engine').setLevel(logging.ERROR)
+
+    settings = get_appsettings(config_uri, options=options)
+    configure_es_from_config(settings)
+
+    client = elasticsearch_config['client']
+    index_name = elasticsearch_config['index']
+
+    # Delete index if it exists (to fix potentially wrong mappings)
+    if client.indices.exists(index=index_name):
+        print('Deleting existing index: {}'.format(index_name))
+        client.indices.delete(index=index_name)
+
+    # Create index with analysis settings
+    print('Creating index: {}'.format(index_name))
+    client.indices.create(index=index_name, body={
+        'settings': {
+            'analysis': analysis_settings
+        }
+    })
+
+    # Apply the correct mapping for every document type
+    # (if we let ES infer types by directly running fill_index.py,
+    # we could run into type errors on some fields like
+    # a mapping geom -> double instead of geo_point).
+    for doc_type, search_class in search_documents.items():
+        print('Applying mapping for doc_type: {}'.format(doc_type))
+        search_class._doc_type.mapping.save(index_name)
+
+    print('Done. Index "{}" created with correct mappings.'.format(index_name))
+    print('You can now run fill_index.py safely.')
+
+
+if __name__ == '__main__':
+    main()

c2corg_api/scripts/es/fill_index.py

@@ -8,7 +8,7 @@
 from pyramid.paster import (
     get_appsettings,
     setup_logging,
-    )
+)
 
 from pyramid.scripts.common import parse_vars
 from datetime import datetime, timedelta
@@ -90,3 +90,7 @@ def progress(count, total_count):
 
     duration = datetime.now() - status['start_time']
     print('Done (duration: {0})'.format(duration))
+
+
+if __name__ == "__main__":
+    main()

c2corg_api/search/advanced_search.py

@@ -1,10 +1,6 @@
 from c2corg_api.search.mapping_types import meta_param_keys
 from c2corg_api.search.search_filters import build_query
 
-# the chunk size for each elastic search request
-# should be <= ES_MAX_RESULT_WINDOW
-CHUNK_SIZE = 10000
-
 
 def get_search_documents(url_params, meta_params, doc_type):
     """Returns a function that when called with a base-query returns all
@@ -89,13 +85,14 @@ def get_all_filtered_docs(
 
 
 def chunk_ids(ids_set):
+    from c2corg_api.views.document import ES_MAX_RESULT_WINDOW
     """
     Yield successive chunks of IDs from a set/list.
     chunk size is ES_MAX_RESULT_WINDOW
     """
     ids_list = list(ids_set)
-    for i in range(0, len(ids_list), CHUNK_SIZE):
-        yield ids_list[i:i + CHUNK_SIZE]
+    for i in range(0, len(ids_list), ES_MAX_RESULT_WINDOW):
+        yield ids_list[i:i + ES_MAX_RESULT_WINDOW]
 
 
 def contains_search_params(url_params):