From 15bc9a4d7f00d0775adf11aa60a0265641b59579 Mon Sep 17 00:00:00 2001 From: "Corey N. Runkel" Date: Wed, 25 Feb 2026 09:57:46 -0500 Subject: [PATCH 1/6] Claude's first pass --- .../controllers/stop_event_controller.ex | 319 ++++++++++ apps/api_web/lib/api_web/router.ex | 2 + .../lib/api_web/views/stop_event_view.ex | 66 ++ .../stop_event_controller_test.exs | 597 ++++++++++++++++++ apps/model/lib/model/stop_event.ex | 64 ++ apps/parse/lib/parse/stop_events.ex | 110 ++++ apps/parse/test/parse/stop_events_test.exs | 154 +++++ apps/state/lib/state.ex | 3 +- apps/state/lib/state/stop_event.ex | 120 ++++ apps/state/test/state/stop_event_test.exs | 228 +++++++ apps/state_mediator/config/config.exs | 5 + apps/state_mediator/lib/state_mediator.ex | 29 +- config/runtime.exs | 5 + 13 files changed, 1700 insertions(+), 2 deletions(-) create mode 100644 apps/api_web/lib/api_web/controllers/stop_event_controller.ex create mode 100644 apps/api_web/lib/api_web/views/stop_event_view.ex create mode 100644 apps/api_web/test/api_web/controllers/stop_event_controller_test.exs create mode 100644 apps/model/lib/model/stop_event.ex create mode 100644 apps/parse/lib/parse/stop_events.ex create mode 100644 apps/parse/test/parse/stop_events_test.exs create mode 100644 apps/state/lib/state/stop_event.ex create mode 100644 apps/state/test/state/stop_event_test.exs diff --git a/apps/api_web/lib/api_web/controllers/stop_event_controller.ex b/apps/api_web/lib/api_web/controllers/stop_event_controller.ex new file mode 100644 index 000000000..a9e2bf9f8 --- /dev/null +++ b/apps/api_web/lib/api_web/controllers/stop_event_controller.ex @@ -0,0 +1,319 @@ +defmodule ApiWeb.StopEventController do + @moduledoc """ + Controller for Stop Events. Filterable by: + + * trip + * stop + * route + * vehicle + * direction_id + """ + use ApiWeb.Web, :api_controller + alias State.StopEvent + + @filters ~w(trip stop route vehicle direction_id) + @includes ~w(trip stop route vehicle) + @pagination_opts [:offset, :limit, :order_by] + @description """ + Stop events represent the actual arrival and departure times of vehicles at stops along their trips. + This is historical data showing when vehicles actually arrived at or departed from stops, as opposed + to predictions or scheduled times. + + Each stop event contains: + - The actual arrival time (as Unix epoch seconds) + - The actual departure time (as Unix epoch seconds) + - The stop sequence number + - Whether the trip was a revenue trip + + Stop events are identified by a composite key of trip_id, route_id, vehicle_id, and stop_id. + """ + + def state_module, do: State.StopEvent + + swagger_path :index do + get(path("stop_event", :index)) + + description(""" + List of stop events. + + #{@description} + """) + + common_index_parameters(__MODULE__, :stop_event) + + include_parameters() + + parameter( + "filter[trip]", + :query, + :string, + "Filter by trip ID. #{comma_separated_list()}.", + example: "73885810" + ) + + parameter( + "filter[stop]", + :query, + :string, + "Filter by stop ID. #{comma_separated_list()}.", + example: "2231" + ) + + parameter( + "filter[route]", + :query, + :string, + "Filter by route ID. #{comma_separated_list()}.", + example: "64" + ) + + parameter( + "filter[vehicle]", + :query, + :string, + "Filter by vehicle ID. #{comma_separated_list()}.", + example: "y2071" + ) + + filter_param(:direction_id) + + consumes("application/vnd.api+json") + produces("application/vnd.api+json") + response(200, "OK", Schema.ref(:StopEvents)) + response(400, "Bad Request", Schema.ref(:BadRequest)) + response(403, "Forbidden", Schema.ref(:Forbidden)) + response(429, "Too Many Requests", Schema.ref(:TooManyRequests)) + end + + def index_data(conn, params) do + with :ok <- Params.validate_includes(params, @includes, conn), + {:ok, filtered} <- Params.filter_params(params, @filters, conn) do + filtered + |> format_filters() + |> StopEvent.filter_by() + |> State.all(pagination_opts(params, conn)) + else + {:error, _, _} = error -> error + end + end + + @spec format_filters(%{optional(String.t()) => String.t()}) :: StopEvent.filters() + defp format_filters(filters, acc \\ %{}) + + defp format_filters(%{"trip" => trip_ids} = filters, acc) do + new_acc = Map.put(acc, :trip_ids, Params.split_on_comma(trip_ids)) + + filters + |> Map.delete("trip") + |> format_filters(new_acc) + end + + defp format_filters(%{"stop" => stop_ids} = filters, acc) do + new_acc = Map.put(acc, :stop_ids, Params.split_on_comma(stop_ids)) + + filters + |> Map.delete("stop") + |> format_filters(new_acc) + end + + defp format_filters(%{"route" => route_ids} = filters, acc) do + new_acc = Map.put(acc, :route_ids, Params.split_on_comma(route_ids)) + + filters + |> Map.delete("route") + |> format_filters(new_acc) + end + + defp format_filters(%{"vehicle" => vehicle_ids} = filters, acc) do + new_acc = Map.put(acc, :vehicle_ids, Params.split_on_comma(vehicle_ids)) + + filters + |> Map.delete("vehicle") + |> format_filters(new_acc) + end + + defp format_filters(%{"direction_id" => direction_id} = filters, acc) do + new_acc = Map.put(acc, :direction_id, Params.direction_id(%{"direction_id" => direction_id})) + + filters + |> Map.delete("direction_id") + |> format_filters(new_acc) + end + + defp format_filters(_filters, acc) do + acc + end + + defp pagination_opts(params, conn) do + opts = + params + |> Params.filter_opts(@pagination_opts, conn) + + if is_list(opts) do + Keyword.put_new(opts, :order_by, {:id, :asc}) + else + opts + |> Map.to_list() + |> Keyword.put_new(:order_by, {:id, :asc}) + end + end + + swagger_path :show do + get(path("stop_event", :show)) + + description(""" + Show a particular stop event by its composite ID. + + #{@description} + """) + + parameter(:id, :path, :string, "Unique identifier for stop event (trip_id-route_id-vehicle_id-stop_id)") + include_parameters() + + consumes("application/vnd.api+json") + produces("application/vnd.api+json") + + response(200, "OK", Schema.ref(:StopEvent)) + response(403, "Forbidden", Schema.ref(:Forbidden)) + response(404, "Not Found", Schema.ref(:NotFound)) + response(429, "Too Many Requests", Schema.ref(:TooManyRequests)) + end + + def show_data(_conn, %{"id" => id}) do + StopEvent.by_id(id) + end + + defp include_parameters(schema) do + ApiWeb.SwaggerHelpers.include_parameters( + schema, + @includes, + description: """ + | include | Description | + |-|-| + | `trip` | The trip associated with this stop event. | + | `stop` | The stop where the event occurred. | + | `route` | The route associated with this stop event. | + | `vehicle` | The vehicle that served this trip. | + """ + ) + end + + def swagger_definitions do + import PhoenixSwagger.JsonApi, except: [page: 1] + + %{ + StopEventResource: + resource do + description(""" + Actual arrival and departure times of vehicles at stops. + """) + + attributes do + vehicle_id( + :string, + """ + The vehicle ID that served this trip. + """, + example: "y2071" + ) + + start_date( + :string, + """ + The service date of the trip in YYYY-MM-DD format. + """, + example: "2026-02-24", + format: :date + ) + + trip_id( + :string, + """ + The trip ID associated with this stop event. + """, + example: "73885810" + ) + + direction_id( + :integer, + """ + Direction in which the trip is traveling: + - `0` - Travel in one direction (e.g. outbound travel) + - `1` - Travel in the opposite direction (e.g. inbound travel) + """, + enum: [0, 1], + example: 0 + ) + + route_id( + :string, + """ + The route ID associated with this stop event. + """, + example: "64" + ) + + start_time( + :string, + """ + The scheduled start time of the trip in HH:MM:SS format. + """, + example: "16:07:00" + ) + + revenue( + :string, + """ + Whether this stop event is for a revenue trip: + - `REVENUE` - A revenue trip + - `NON_REVENUE` - A non-revenue trip + """, + enum: ["REVENUE", "NON_REVENUE"], + example: "REVENUE" + ) + + stop_id( + :string, + """ + The stop ID where the event occurred. + """, + example: "2231" + ) + + current_stop_sequence( + :integer, + """ + The stop sequence number along the trip. Increases monotonically but values need not be consecutive. + """, + example: 1 + ) + + arrived( + [:integer, :null], + """ + When the vehicle arrived at the stop, as seconds since Unix epoch (UTC). `null` if the first stop on the trip. + """, + example: 1771966486, + "x-nullable": true + ) + + departed( + [:integer, :null], + """ + When the vehicle departed from the stop, as seconds since Unix epoch (UTC). `null` if the last stop on the trip or if the vehicle has not yet departed. + """, + example: 1771967246, + "x-nullable": true + ) + end + + relationship(:trip) + relationship(:stop) + relationship(:route) + relationship(:vehicle) + end, + StopEvents: page(:StopEventResource), + StopEvent: single(:StopEventResource) + } + end +end diff --git a/apps/api_web/lib/api_web/router.ex b/apps/api_web/lib/api_web/router.ex index 8963943cc..96f404e23 100644 --- a/apps/api_web/lib/api_web/router.ex +++ b/apps/api_web/lib/api_web/router.ex @@ -100,6 +100,8 @@ defmodule ApiWeb.Router do resources("/live_facilities", LiveFacilityController, only: [:index, :show]) resources("/live-facilities", LiveFacilityController, only: [:index, :show]) resources("/services", ServiceController, only: [:index, :show]) + resources("/stop-events", StopEventController, only: [:index, :show]) + resources("/stop_events", StopEventController, only: [:index, :show]) end scope "/docs/swagger" do diff --git a/apps/api_web/lib/api_web/views/stop_event_view.ex b/apps/api_web/lib/api_web/views/stop_event_view.ex new file mode 100644 index 000000000..8cfc0ba6e --- /dev/null +++ b/apps/api_web/lib/api_web/views/stop_event_view.ex @@ -0,0 +1,66 @@ +defmodule ApiWeb.StopEventView do + use ApiWeb.Web, :api_view + + location(:stop_event_location) + + def stop_event_location(stop_event, conn), + do: stop_event_path(conn, :show, stop_event.id) + + has_one( + :trip, + type: :trip, + serializer: ApiWeb.TripView, + field: :trip_id + ) + + has_one( + :stop, + type: :stop, + serializer: ApiWeb.StopView, + field: :stop_id + ) + + has_one( + :route, + type: :route, + serializer: ApiWeb.RouteView, + field: :route_id + ) + + has_one( + :vehicle, + type: :vehicle, + serializer: ApiWeb.VehicleView, + field: :vehicle_id + ) + + attributes([ + :vehicle_id, + :start_date, + :trip_id, + :direction_id, + :route_id, + :start_time, + :revenue, + :stop_id, + :current_stop_sequence, + :arrived, + :departed + ]) + + def trip(%{trip_id: trip_id}, conn) do + optional_relationship("trip", trip_id, &State.Trip.by_primary_id/1, conn) + end + + def stop(%{stop_id: stop_id}, conn) do + optional_relationship("stop", stop_id, &State.Stop.by_id/1, conn) + end + + def route(%{route_id: route_id}, conn) do + optional_relationship("route", route_id, &State.Route.by_id/1, conn) + end + + def vehicle(%{vehicle_id: vehicle_id}, conn) do + optional_relationship("vehicle", vehicle_id, &State.Vehicle.by_id/1, conn) + end +end diff --git a/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs b/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs new file mode 100644 index 000000000..763fc3aca --- /dev/null +++ b/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs @@ -0,0 +1,597 @@ +defmodule ApiWeb.StopEventControllerTest do + @moduledoc false + use ApiWeb.ConnCase + + alias Model.StopEvent + + setup %{conn: conn} do + {:ok, conn: put_req_header(conn, "accept", "application/json")} + end + + describe "index_data/2" do + test "lists all entries on index", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + start_date: ~D[2026-02-24], + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + start_time: "10:00:00", + revenue: :REVENUE, + stop_id: "stop1", + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + }, + %StopEvent{ + id: "trip2-route2-v2-stop2", + vehicle_id: "v2", + start_date: ~D[2026-02-24], + trip_id: "trip2", + direction_id: 1, + route_id: "route2", + start_time: "11:00:00", + revenue: :NON_REVENUE, + stop_id: "stop2", + current_stop_sequence: 1, + arrived: 1771968343, + departed: nil + } + ]) + + conn = get(conn, stop_event_path(conn, :index)) + response = json_response(conn, 200) + + assert [ + %{"type" => "stop_event"}, + %{"type" => "stop_event"} + ] = response["data"] + end + + test "conforms to swagger response", %{swagger_schema: schema, conn: conn} do + stop_event = %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + start_date: ~D[2026-02-24], + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + start_time: "10:00:00", + revenue: :REVENUE, + stop_id: "stop1", + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + } + + State.StopEvent.new_state([stop_event]) + + response = get(conn, stop_event_path(conn, :index)) + assert validate_resp_schema(response, schema, "StopEvents") + end + + test "can filter by trip", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + }, + %StopEvent{ + id: "trip2-route2-stop2", + trip_id: "trip2", + direction_id: 0, + route_id: "route2", + stop_id: "stop2", + start_date: ~D[2026-02-24], + start_time: "11:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771968343, + departed: nil + } + ]) + + conn = get(conn, stop_event_path(conn, :index, %{"filter" => %{"trip" => "trip1"}})) + + assert [%{"id" => "trip1-route1-v1-stop1"}] = json_response(conn, 200)["data"] + end + + test "can filter by stop", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + }, + %StopEvent{ + id: "trip1-route1-v1-stop2", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop2", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 2, + arrived: 1771967286, + departed: 1771967333 + } + ]) + + conn = get(conn, stop_event_path(conn, :index, %{"filter" => %{"stop" => "stop2"}})) + + assert [%{"id" => "trip1-route1-v1-stop2"}] = json_response(conn, 200)["data"] + end + + test "can filter by route", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + }, + %StopEvent{ + id: "trip2-route2-stop2", + trip_id: "trip2", + direction_id: 0, + route_id: "route2", + stop_id: "stop2", + start_date: ~D[2026-02-24], + start_time: "11:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771968343, + departed: nil + } + ]) + + conn = get(conn, stop_event_path(conn, :index, %{"filter" => %{"route" => "route1"}})) + + assert [%{"id" => "trip1-route1-v1-stop1"}] = json_response(conn, 200)["data"] + end + + test "can filter by direction_id", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + }, + %StopEvent{ + id: "trip2-route2-stop2", + trip_id: "trip2", + direction_id: 1, + route_id: "route2", + stop_id: "stop2", + start_date: ~D[2026-02-24], + start_time: "11:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771968343, + departed: nil + } + ]) + + conn = + get(conn, stop_event_path(conn, :index, %{"filter" => %{"direction_id" => "0"}})) + + assert [%{"id" => "trip1-route1-v1-stop1"}] = json_response(conn, 200)["data"] + end + + test "can filter by route and direction_id simultaneously", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + }, + %StopEvent{ + id: "trip2-route1-v2-stop2", + vehicle_id: "v2", + trip_id: "trip2", + direction_id: 1, + route_id: "route1", + stop_id: "stop2", + start_date: ~D[2026-02-24], + start_time: "11:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771968343, + departed: nil + }, + %StopEvent{ + id: "trip3-route2-v3-stop3", + vehicle_id: "v3", + trip_id: "trip3", + direction_id: 0, + route_id: "route2", + stop_id: "stop3", + start_date: ~D[2026-02-24], + start_time: "12:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771969000, + departed: 1771969100 + } + ]) + + conn = + get( + conn, + stop_event_path(conn, :index, %{ + "filter" => %{"route" => "route1", "direction_id" => "0"} + }) + ) + + assert [%{"id" => "trip1-route1-v1-stop1"}] = json_response(conn, 200)["data"] + end + + test "can filter by trip and stop simultaneously", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + }, + %StopEvent{ + id: "trip1-route1-v1-stop2", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop2", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 2, + arrived: 1771967286, + departed: 1771967333 + }, + %StopEvent{ + id: "trip2-route1-v2-stop2", + vehicle_id: "v2", + trip_id: "trip2", + direction_id: 0, + route_id: "route1", + stop_id: "stop2", + start_date: ~D[2026-02-24], + start_time: "11:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771968343, + departed: nil + } + ]) + + conn = + get( + conn, + stop_event_path(conn, :index, %{"filter" => %{"trip" => "trip1", "stop" => "stop2"}}) + ) + + assert [%{"id" => "trip1-route1-v1-stop2"}] = json_response(conn, 200)["data"] + end + + test "can filter by route, stop, and direction_id simultaneously", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + }, + %StopEvent{ + id: "trip2-route1-stop1", + trip_id: "trip2", + direction_id: 1, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "11:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771968343, + departed: nil + }, + %StopEvent{ + id: "trip3-route1-v3-stop2", + vehicle_id: "v3", + trip_id: "trip3", + direction_id: 0, + route_id: "route1", + stop_id: "stop2", + start_date: ~D[2026-02-24], + start_time: "12:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771969000, + departed: 1771969100 + }, + %StopEvent{ + id: "trip4-route2-v4-stop1", + vehicle_id: "v4", + trip_id: "trip4", + direction_id: 0, + route_id: "route2", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "13:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771970000, + departed: 1771970200 + } + ]) + + conn = + get( + conn, + stop_event_path(conn, :index, %{ + "filter" => %{"route" => "route1", "stop" => "stop1", "direction_id" => "0"} + }) + ) + + assert [%{"id" => "trip1-route1-v1-stop1"}] = json_response(conn, 200)["data"] + end + + test "can filter by multiple trips, routes, and stops simultaneously", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + }, + %StopEvent{ + id: "trip2-route1-v2-stop2", + vehicle_id: "v2", + trip_id: "trip2", + direction_id: 0, + route_id: "route1", + stop_id: "stop2", + start_date: ~D[2026-02-24], + start_time: "11:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771968343, + departed: nil + }, + %StopEvent{ + id: "trip3-route2-v3-stop3", + vehicle_id: "v3", + trip_id: "trip3", + direction_id: 0, + route_id: "route2", + stop_id: "stop3", + start_date: ~D[2026-02-24], + start_time: "12:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771969000, + departed: 1771969100 + }, + %StopEvent{ + id: "trip2-route2-v2-stop1", + vehicle_id: "v2", + trip_id: "trip2", + direction_id: 1, + route_id: "route2", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "13:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771970000, + departed: 1771970200 + } + ]) + + conn = + get( + conn, + stop_event_path(conn, :index, %{ + "filter" => %{"trip" => "trip1,trip2", "route" => "route1,route2", "stop" => "stop1"} + }) + ) + + response = json_response(conn, 200)["data"] + ids = Enum.map(response, & &1["id"]) |> Enum.sort() + # Both trip1-route1-stop1 and trip2-route2-stop1 match the filters + assert ids == ["trip1-route1-v1-stop1", "trip2-route2-v2-stop1"] + end + + test "returns empty when filters match no records", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + } + ]) + + conn = + get( + conn, + stop_event_path(conn, :index, %{ + "filter" => %{"route" => "route1", "direction_id" => "1"} + }) + ) + + assert [] = json_response(conn, 200)["data"] + end + + test "pagination works", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + }, + %StopEvent{ + id: "trip1-route1-v1-stop2", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop2", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 2, + arrived: 1771967286, + departed: 1771967333 + } + ]) + + conn = + get(conn, stop_event_path(conn, :index, %{"page" => %{"limit" => "1", "offset" => "0"}})) + + response = json_response(conn, 200) + assert length(response["data"]) == 1 + assert response["links"]["next"] + end + end + + describe "show_data/2" do + test "shows chosen resource", %{conn: conn} do + stop_event = %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + start_date: ~D[2026-02-24], + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + start_time: "10:00:00", + revenue: :REVENUE, + stop_id: "stop1", + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + } + + State.StopEvent.new_state([stop_event]) + + conn = get(conn, stop_event_path(conn, :show, stop_event.id)) + assert json_response(conn, 200)["data"]["id"] == stop_event.id + end + + test "conforms to swagger response", %{swagger_schema: schema, conn: conn} do + stop_event = %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + start_date: ~D[2026-02-24], + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + start_time: "10:00:00", + revenue: :REVENUE, + stop_id: "stop1", + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + } + + State.StopEvent.new_state([stop_event]) + + response = get(conn, stop_event_path(conn, :show, stop_event.id)) + assert validate_resp_schema(response, schema, "StopEvent") + end + + test "does not show resource when id is nonexistent", %{conn: conn} do + conn = get(conn, stop_event_path(conn, :show, "nonexistent")) + assert json_response(conn, 404) + end + end +end diff --git a/apps/model/lib/model/stop_event.ex b/apps/model/lib/model/stop_event.ex new file mode 100644 index 000000000..7ed43e48d --- /dev/null +++ b/apps/model/lib/model/stop_event.ex @@ -0,0 +1,64 @@ +defmodule Model.StopEvent do + @moduledoc """ + The actual `arrival_time` and `departure_time` of a `vehicle_id` to/from a `stop_sequence` in a `trip_id`. + along a trip (`trip_id`) going a direction (`direction_id`) along a route (`route_id`). This is the actual time a vehicle arrived at or departed from a stop, as opposed to a prediction of when a vehicle will arrive at or depart from a stop (`Model.Prediction.t`) or the scheduled time of arrival or departure (`Model.Schedule.t`). + + For the predicted times, see `Model.Prediction.t`. + For the scheduled times, see `Model.Schedule.t`. + """ + + use Recordable, [ + :id, + :vehicle_id, + :start_date, + :trip_id, + :direction_id, + :route_id, + :start_time, + :revenue, + :stop_id, + :current_stop_sequence, + :arrived, + :departed + ] + + @typedoc """ + * `:id` - Composite key: `{trip_id}-{route_id}-{vehicle_id}-{stop_id}`. + * `:vehicle_id` - The vehicle serving this trip. See + [GTFS Realtime `FeedMessage` `FeedEntity` `VehiclePosition` `VehicleDescriptor` `id`](https://github.com/google/transit/blob/master/gtfs-realtime/spec/en/reference.md#message-vehicledescriptor). + * `:start_date` - The service date of the `trip_id`. + * `:trip_id` - The trip the `stop_id` is on. See [GTFS Realtime `FeedMesage` `FeedEntity` `TripUpdate` `TripDescriptor`](https://github.com/google/transit/blob/master/gtfs-realtime/spec/en/reference.md#message-tripdescriptor) + * `:direction_id` - Which direction along `route_id` the `trip_id` is going. See + [GTFS `trips.txt` `direction_id`](https://github.com/google/transit/blob/master/gtfs/spec/en/reference.md#tripstxt). + * `:route_id` - The route `trip_id` is on doing in `direction_id`. See + [GTFS `trips.txt` `route_id`](https://github.com/google/transit/blob/master/gtfs/spec/en/reference.md#tripstxt) + * `start_time` - The time the `trip_id` was scheduled to start. + * `:revenue` - Whether or not the stop event is for a revenue trip. + * `:stop_id` - Stop associated with arrived/departed. See + [GTFS Realtime `FeedMesage` `FeedEntity` `TripUpdate` `StopTimeUpdate` `stop_id`](https://github.com/google/transit/blob/master/gtfs-realtime/spec/en/reference.md#message-stoptimeupdate). + * `:current_stop_sequence` - The sequence of the stop along the `trip_id`. The stop sequence increases monotonically but values need not be consecutive. + See [GTFS Realtime `FeedMesage` `FeedEntity` `VehiclePosition` `current_stop_sequence`](https://github.com/google/transit/blob/master/gtfs-realtime/spec/en/reference.md#message-vehicleposition). + * `:arrived` - When the vehicle arrived at the stop as seconds since Unix epoch in timezone `America/New_York`. `nil` if the first stop (`stop_id`) on the `trip_id`. + * `:departed` - When the vehicle arrived at the stop as seconds since Unix epoch in timezone `America/New_York`. `nil` if the last stop (`stop_id`) on the `trip_id` + """ + @type t :: %__MODULE__{ + id: String.t(), + vehicle_id: String.t(), + start_date: Date.t(), + trip_id: Model.Trip.id(), + direction_id: Model.Direction.id(), + route_id: Model.Route.id(), + start_time: non_neg_integer, + stop_id: Model.Stop.id(), + current_stop_sequence: non_neg_integer, + revenue: :REVENUE | :NON_REVENUE, + arrived: DateTime.t() | nil, + departed: DateTime.t() | nil + } + + @spec trip_id(t) :: Model.Trip.id() + def trip_id(%__MODULE__{trip_id: trip_id}), do: trip_id + + @spec vehicle_id(t) :: String.t() + def vehicle_id(%__MODULE__{vehicle_id: vehicle_id}), do: vehicle_id +end diff --git a/apps/parse/lib/parse/stop_events.ex b/apps/parse/lib/parse/stop_events.ex new file mode 100644 index 000000000..dfc41c626 --- /dev/null +++ b/apps/parse/lib/parse/stop_events.ex @@ -0,0 +1,110 @@ +defmodule Parse.StopEvents do + @moduledoc """ + Parser for the Stop Events data from S3 (NDJSON format) + """ + + require Logger + + @behaviour Parse + + @impl Parse + def parse(binary) do + binary + |> String.split("\n", trim: true) + |> Enum.flat_map(&parse_line/1) + end + + defp parse_line(line) do + case Jason.decode(line) do + {:ok, record} -> + parse_record(record) + + e -> + Logger.warning("#{__MODULE__} decode_error e=#{inspect(e)}") + [] + end + end + + defp parse_record(%{ + "start_date" => start_date, + "trip_id" => trip_id, + "vehicle_id" => vehicle_id, + "direction_id" => direction_id, + "route_id" => route_id, + "start_time" => start_time, + "revenue" => revenue, + "stop_events" => stop_events + }) + when is_list(stop_events) do + with {:ok, date} <- parse_date(start_date), + {:ok, revenue_atom} <- parse_revenue(revenue) do + Enum.flat_map(stop_events, fn stop_event -> + parse_stop_event(stop_event, %{ + start_date: date, + trip_id: trip_id, + vehicle_id: vehicle_id, + direction_id: direction_id, + route_id: route_id, + start_time: start_time, + revenue: revenue_atom + }) + end) + else + error -> + Logger.warning("#{__MODULE__} parse_error error=#{inspect(error)} trip_id=#{trip_id}") + [] + end + end + + defp parse_record(record) do + Logger.warning("#{__MODULE__} parse_error error=missing_fields #{inspect(record)}") + [] + end + + defp parse_stop_event( + %{ + "stop_id" => stop_id, + "current_stop_sequence" => current_stop_sequence, + "arrived" => arrived, + "departed" => departed + }, + trip_data + ) do + id = "#{trip_data.trip_id}-#{trip_data.route_id}-#{trip_data.vehicle_id}-#{stop_id}" + + [ + %Model.StopEvent{ + id: id, + vehicle_id: trip_data.vehicle_id, + start_date: trip_data.start_date, + trip_id: trip_data.trip_id, + direction_id: trip_data.direction_id, + route_id: trip_data.route_id, + start_time: trip_data.start_time, + revenue: trip_data.revenue, + stop_id: stop_id, + current_stop_sequence: current_stop_sequence, + arrived: arrived, + departed: departed + } + ] + end + + defp parse_stop_event(stop_event, _trip_data) do + Logger.warning("#{__MODULE__} parse_error error=missing_fields #{inspect(stop_event)}") + [] + end + + defp parse_date(<>) do + case Date.new(String.to_integer(year), String.to_integer(month), String.to_integer(day)) do + {:ok, date} -> {:ok, date} + _ -> {:error, :invalid_date} + end + end + + defp parse_date(_), do: {:error, :invalid_date} + + defp parse_revenue(true), do: {:ok, :REVENUE} + defp parse_revenue(false), do: {:ok, :NON_REVENUE} + defp parse_revenue(_), do: {:error, :invalid_revenue} +end diff --git a/apps/parse/test/parse/stop_events_test.exs b/apps/parse/test/parse/stop_events_test.exs new file mode 100644 index 000000000..142257a1d --- /dev/null +++ b/apps/parse/test/parse/stop_events_test.exs @@ -0,0 +1,154 @@ +defmodule Parse.StopEventsTest do + use ExUnit.Case + import ExUnit.CaptureLog + + import Parse.StopEvents + alias Model.StopEvent + + describe "parse" do + test "parses valid NDJSON data with multiple stop events" do + ndjson = """ + {"id":"73885810-64-y2071","timestamp":1771968343,"start_date":"20260224","trip_id":"73885810","vehicle_id":"y2071","direction_id":0,"route_id":"64","start_time":"16:07:00","revenue":true,"stop_events":[{"stop_id":"2231","current_stop_sequence":1,"arrived":1771966486,"departed":1771967246},{"stop_id":"12232","current_stop_sequence":2,"arrived":1771967286,"departed":1771967333}]} + {"id":"73221192-Green-E-G-10077","timestamp":1771950045,"start_date":"20260224","trip_id":"73221192","vehicle_id":"G-10077","direction_id":0,"route_id":"Green-E","start_time":"10:16:00","revenue":true,"stop_events":[{"stop_id":"70512","current_stop_sequence":4,"arrived":1771946303,"departed":1771946479}]} + """ + + result = parse(ndjson) + + assert length(result) == 3 + + assert %StopEvent{ + id: "73885810-64-y2071-2231", + vehicle_id: "y2071", + start_date: ~D[2026-02-24], + trip_id: "73885810", + direction_id: 0, + route_id: "64", + start_time: "16:07:00", + revenue: :REVENUE, + stop_id: "2231", + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + } in result + + assert %StopEvent{ + id: "73885810-64-y2071-12232", + vehicle_id: "y2071", + start_date: ~D[2026-02-24], + trip_id: "73885810", + direction_id: 0, + route_id: "64", + start_time: "16:07:00", + revenue: :REVENUE, + stop_id: "12232", + current_stop_sequence: 2, + arrived: 1771967286, + departed: 1771967333 + } in result + + assert %StopEvent{ + id: "73221192-Green-E-G-10077-70512", + vehicle_id: "G-10077", + start_date: ~D[2026-02-24], + trip_id: "73221192", + direction_id: 0, + route_id: "Green-E", + start_time: "10:16:00", + revenue: :REVENUE, + stop_id: "70512", + current_stop_sequence: 4, + arrived: 1771946303, + departed: 1771946479 + } in result + end + + test "handles null departed times for last stop" do + ndjson = """ + {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","current_stop_sequence":1,"arrived":1771966486,"departed":null}]} + """ + + result = parse(ndjson) + + assert [%StopEvent{departed: nil}] = result + end + + test "handles null arrived times for first stop" do + ndjson = """ + {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","current_stop_sequence":1,"arrived":null,"departed":1771967246}]} + """ + + result = parse(ndjson) + + assert [%StopEvent{arrived: nil}] = result + end + + test "handles non-revenue trips" do + ndjson = """ + {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":false,"stop_events":[{"stop_id":"stop1","current_stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + """ + + result = parse(ndjson) + + assert [%StopEvent{revenue: :NON_REVENUE}] = result + end + + test "ignores empty lines in NDJSON" do + ndjson = """ + + {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","current_stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + + """ + + result = parse(ndjson) + + assert length(result) == 1 + end + + test "logs and ignores lines with missing required fields" do + ndjson = """ + {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","current_stop_sequence":1}]} + {"id":"valid","timestamp":1771968343,"start_date":"20260224","trip_id":"valid","vehicle_id":"v2","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop2","current_stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + """ + + log = + capture_log(fn -> + result = parse(ndjson) + assert length(result) == 1 + assert [%StopEvent{trip_id: "valid"}] = result + end) + + assert log =~ "missing_fields" + end + + test "logs and ignores lines with invalid date format" do + ndjson = """ + {"id":"test-trip","timestamp":1771968343,"start_date":"invalid","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","current_stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + """ + + log = + capture_log(fn -> + assert parse(ndjson) == [] + end) + + assert log =~ "invalid_date" + end + + test "handles invalid JSON" do + log = + capture_log(fn -> + assert parse("{abc\n{def}") == [] + end) + + assert log =~ "decode_error" + end + + test "handles trip with empty stop_events array" do + ndjson = """ + {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[]} + """ + + result = parse(ndjson) + assert result == [] + end + end +end diff --git a/apps/state/lib/state.ex b/apps/state/lib/state.ex index 3638c4dc5..a4d01c72e 100644 --- a/apps/state/lib/state.ex +++ b/apps/state/lib/state.ex @@ -36,7 +36,8 @@ defmodule State do State.RoutesByService, State.Shape, State.Feed, - State.CommuterRailOccupancy + State.CommuterRailOccupancy, + State.StopEvent ] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html diff --git a/apps/state/lib/state/stop_event.ex b/apps/state/lib/state/stop_event.ex new file mode 100644 index 000000000..7559c61d7 --- /dev/null +++ b/apps/state/lib/state/stop_event.ex @@ -0,0 +1,120 @@ +defmodule State.StopEvent do + @moduledoc """ + State for stop events - actual arrival/departure times of vehicles at stops + """ + use State.Server, + indices: [:id, :trip_id, :stop_id, :route_id, :vehicle_id], + parser: Parse.StopEvents, + recordable: Model.StopEvent + + alias Model.Route + alias Model.StopEvent + alias Model.Stop + alias Model.Trip + + @type filters :: %{ + optional(:trip_ids) => [Trip.id()], + optional(:stop_ids) => [Stop.id()], + optional(:route_ids) => [Route.id()], + optional(:vehicle_ids) => [String.t()], + optional(:direction_id) => 0 | 1 + } + + @spec by_id(String.t()) :: StopEvent.t() | nil + def by_id(id) do + case super(id) do + [] -> nil + [stop_event] -> stop_event + end + end + + @spec filter_by(filters()) :: [StopEvent.t()] + def filter_by(%{trip_ids: trip_ids} = filters) when is_list(trip_ids) and trip_ids != [] do + trip_ids + |> by_trip_ids() + |> apply_additional_filters(Map.delete(filters, :trip_ids)) + end + + def filter_by(%{stop_ids: stop_ids} = filters) when is_list(stop_ids) and stop_ids != [] do + stop_ids + |> by_stop_ids() + |> apply_additional_filters(Map.delete(filters, :stop_ids)) + end + + def filter_by(%{route_ids: route_ids} = filters) when is_list(route_ids) and route_ids != [] do + route_ids + |> by_route_ids() + |> apply_additional_filters(Map.delete(filters, :route_ids)) + end + + def filter_by(%{vehicle_ids: vehicle_ids} = filters) + when is_list(vehicle_ids) and vehicle_ids != [] do + vehicle_ids + |> by_vehicle_ids() + |> apply_additional_filters(Map.delete(filters, :vehicle_ids)) + end + + def filter_by(%{direction_id: direction_id}) do + all() + |> Enum.filter(fn %StopEvent{direction_id: d_id} -> d_id == direction_id end) + end + + def filter_by(%{} = map) when map_size(map) == 0 do + all() + end + + def filter_by(_filters) do + [] + end + + defp apply_additional_filters(events, %{trip_ids: trip_ids} = filters) + when is_list(trip_ids) do + trip_id_set = MapSet.new(trip_ids) + + Enum.filter(events, fn %StopEvent{trip_id: trip_id} -> + MapSet.member?(trip_id_set, trip_id) + end) + |> apply_additional_filters(Map.delete(filters, :trip_ids)) + end + + defp apply_additional_filters(events, %{stop_ids: stop_ids} = filters) + when is_list(stop_ids) do + stop_id_set = MapSet.new(stop_ids) + + Enum.filter(events, fn %StopEvent{stop_id: stop_id} -> + MapSet.member?(stop_id_set, stop_id) + end) + |> apply_additional_filters(Map.delete(filters, :stop_ids)) + end + + defp apply_additional_filters(events, %{route_ids: route_ids} = filters) + when is_list(route_ids) do + route_id_set = MapSet.new(route_ids) + + Enum.filter(events, fn %StopEvent{route_id: route_id} -> + MapSet.member?(route_id_set, route_id) + end) + |> apply_additional_filters(Map.delete(filters, :route_ids)) + end + + defp apply_additional_filters(events, %{vehicle_ids: vehicle_ids} = filters) + when is_list(vehicle_ids) do + vehicle_id_set = MapSet.new(vehicle_ids) + + Enum.filter(events, fn %StopEvent{vehicle_id: vehicle_id} -> + MapSet.member?(vehicle_id_set, vehicle_id) + end) + |> apply_additional_filters(Map.delete(filters, :vehicle_ids)) + end + + defp apply_additional_filters(events, %{direction_id: direction_id} = filters) do + Enum.filter(events, fn %StopEvent{direction_id: d_id} -> + d_id == direction_id + end) + |> apply_additional_filters(Map.delete(filters, :direction_id)) + end + + defp apply_additional_filters(events, _filters) do + events + end +end diff --git a/apps/state/test/state/stop_event_test.exs b/apps/state/test/state/stop_event_test.exs new file mode 100644 index 000000000..0a07574f4 --- /dev/null +++ b/apps/state/test/state/stop_event_test.exs @@ -0,0 +1,228 @@ +defmodule State.StopEventTest do + use ExUnit.Case + + alias Model.StopEvent + import State.StopEvent + + describe "filter_by/1" do + setup do + stop_event1 = %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + start_date: ~D[2026-02-24], + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + start_time: "10:00:00", + revenue: :REVENUE, + stop_id: "stop1", + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + } + + stop_event2 = %StopEvent{ + id: "trip1-route1-v1-stop2", + vehicle_id: "v1", + start_date: ~D[2026-02-24], + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + start_time: "10:00:00", + revenue: :REVENUE, + stop_id: "stop2", + current_stop_sequence: 2, + arrived: 1771967286, + departed: 1771967333 + } + + stop_event3 = %StopEvent{ + id: "trip2-route2-v2-stop3", + vehicle_id: "v2", + start_date: ~D[2026-02-24], + trip_id: "trip2", + direction_id: 1, + route_id: "route2", + start_time: "11:00:00", + revenue: :NON_REVENUE, + stop_id: "stop3", + current_stop_sequence: 1, + arrived: 1771968343, + departed: nil + } + + State.StopEvent.new_state([stop_event1, stop_event2, stop_event3]) + + {:ok, %{event1: stop_event1, event2: stop_event2, event3: stop_event3}} + end + + test "returns all events with empty filters", %{event1: e1, event2: e2, event3: e3} do + result = filter_by(%{}) + assert length(result) == 3 + assert e1 in result + assert e2 in result + assert e3 in result + end + + test "filters by trip_id", %{event1: e1, event2: e2, event3: e3} do + result = filter_by(%{trip_ids: ["trip1"]}) + assert length(result) == 2 + assert e1 in result + assert e2 in result + refute e3 in result + end + + test "filters by multiple trip_ids", %{event1: e1, event2: e2, event3: e3} do + result = filter_by(%{trip_ids: ["trip1", "trip2"]}) + assert length(result) == 3 + assert e1 in result + assert e2 in result + assert e3 in result + end + + test "filters by stop_id", %{event1: e1, event2: e2} do + result = filter_by(%{stop_ids: ["stop1"]}) + assert result == [e1] + end + + test "filters by multiple stop_ids", %{event1: e1, event3: e3} do + result = filter_by(%{stop_ids: ["stop1", "stop3"]}) + assert length(result) == 2 + assert e1 in result + assert e3 in result + end + + test "filters by route_id", %{event1: e1, event2: e2, event3: e3} do + result = filter_by(%{route_ids: ["route1"]}) + assert length(result) == 2 + assert e1 in result + assert e2 in result + refute e3 in result + end + + test "filters by multiple route_ids", %{event1: e1, event2: e2, event3: e3} do + result = filter_by(%{route_ids: ["route1", "route2"]}) + assert length(result) == 3 + assert e1 in result + assert e2 in result + assert e3 in result + end + + test "filters by direction_id", %{event1: e1, event2: e2, event3: e3} do + result = filter_by(%{direction_id: 0}) + assert length(result) == 2 + assert e1 in result + assert e2 in result + refute e3 in result + + result = filter_by(%{direction_id: 1}) + assert result == [e3] + end + + test "filters by trip_id and stop_id", %{event1: e1} do + result = filter_by(%{trip_ids: ["trip1"], stop_ids: ["stop1"]}) + assert result == [e1] + end + + test "filters by route_id and direction_id", %{event1: e1, event2: e2} do + result = filter_by(%{route_ids: ["route1"], direction_id: 0}) + assert length(result) == 2 + assert e1 in result + assert e2 in result + end + + test "filters by trip_id, stop_id, and direction_id simultaneously", %{event1: e1} do + result = filter_by(%{trip_ids: ["trip1"], stop_ids: ["stop1"], direction_id: 0}) + assert result == [e1] + end + + test "filters by route_id, stop_id, and direction_id simultaneously", %{event1: e1} do + result = filter_by(%{route_ids: ["route1"], stop_ids: ["stop1"], direction_id: 0}) + assert result == [e1] + end + + test "filters by multiple values across all filter types" do + # Add more test data for this test + stop_event4 = %StopEvent{ + id: "trip2-route1-v2-stop1", + vehicle_id: "v2", + start_date: ~D[2026-02-24], + trip_id: "trip2", + direction_id: 0, + route_id: "route1", + start_time: "12:00:00", + revenue: :REVENUE, + stop_id: "stop1", + current_stop_sequence: 1, + arrived: 1771969000, + departed: 1771969100 + } + + all_events = State.StopEvent.all() + State.StopEvent.new_state(all_events ++ [stop_event4]) + + # Filter for trip1 OR trip2, route1, stop1, direction 0 + result = + filter_by(%{ + trip_ids: ["trip1", "trip2"], + route_ids: ["route1"], + stop_ids: ["stop1"], + direction_id: 0 + }) + + # Should return both trip1-route1-stop1 and trip2-route1-stop1 + assert length(result) == 2 + assert Enum.all?(result, fn e -> e.route_id == "route1" end) + assert Enum.all?(result, fn e -> e.stop_id == "stop1" end) + assert Enum.all?(result, fn e -> e.direction_id == 0 end) + assert Enum.all?(result, fn e -> e.trip_id in ["trip1", "trip2"] end) + end + + test "returns empty when combining filters that match no records", %{event1: e1, event2: e2} do + # event1 and event2 both have route1, but only event1 has stop1 + # Filtering for route1, stop2, and direction_id 1 should return nothing + result = filter_by(%{route_ids: ["route1"], stop_ids: ["stop2"], direction_id: 1}) + assert result == [] + end + + test "returns empty list for non-matching filters" do + assert filter_by(%{trip_ids: ["nonexistent"]}) == [] + assert filter_by(%{stop_ids: ["nonexistent"]}) == [] + assert filter_by(%{route_ids: ["nonexistent"]}) == [] + assert filter_by(%{direction_id: 2}) == [] + end + + test "returns empty list for empty id lists" do + assert filter_by(%{trip_ids: []}) == [] + assert filter_by(%{stop_ids: []}) == [] + assert filter_by(%{route_ids: []}) == [] + end + end + + describe "by_id/1" do + test "returns stop event by id" do + stop_event = %StopEvent{ + id: "trip1-route1-v1-stop1", + vehicle_id: "v1", + start_date: ~D[2026-02-24], + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + start_time: "10:00:00", + revenue: :REVENUE, + stop_id: "stop1", + current_stop_sequence: 1, + arrived: 1771966486, + departed: 1771967246 + } + + State.StopEvent.new_state([stop_event]) + + assert by_id("trip1-route1-v1-stop1") == stop_event + end + + test "returns nil for non-existent id" do + assert by_id("nonexistent") == nil + end + end +end diff --git a/apps/state_mediator/config/config.exs b/apps/state_mediator/config/config.exs index 56731792c..217911547 100644 --- a/apps/state_mediator/config/config.exs +++ b/apps/state_mediator/config/config.exs @@ -15,6 +15,11 @@ config :state_mediator, :commuter_rail_crowding, s3_object: {:system, "CR_CROWDING_S3_OBJECT"}, source: {:system, "CR_CROWING_SOURCE", "s3"} +config :state_mediator, :stop_events, + enabled: {:system, "STOP_EVENTS_ENABLED", "false"}, + s3_bucket: {:system, "STOP_EVENTS_S3_BUCKET"}, + s3_object: {:system, "STOP_EVENTS_S3_OBJECT"} + config :state_mediator, Realtime, gtfs_url: {:system, "MBTA_GTFS_URL", "https://cdn.mbta.com/MBTA_GTFS.zip"}, alert_url: {:system, "ALERT_URL", "https://cdn.mbta.com/realtime/Alerts_enhanced.json"} diff --git a/apps/state_mediator/lib/state_mediator.ex b/apps/state_mediator/lib/state_mediator.ex index 64bff4a5d..8ff16ecca 100644 --- a/apps/state_mediator/lib/state_mediator.ex +++ b/apps/state_mediator/lib/state_mediator.ex @@ -14,7 +14,8 @@ defmodule StateMediator do crowding_children( app_value(:commuter_rail_crowding, :enabled) == "true", crowding_source - ) + ) ++ + stop_event_children(app_value(:stop_events, :enabled) == "true") # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options @@ -155,6 +156,32 @@ defmodule StateMediator do [] end + @spec stop_event_children(boolean()) :: [ + :supervisor.child_spec() | {module(), term()} | module() + ] + defp stop_event_children(true) do + Logger.info("#{__MODULE__} STOP_EVENTS_ENABLED=true") + + [ + { + StateMediator.S3Mediator, + [ + spec_id: :stop_event_mediator, + bucket_arn: app_value(:stop_events, :s3_bucket), + object: app_value(:stop_events, :s3_object), + interval: 60 * 1_000, + sync_timeout: 30_000, + state: State.StopEvent + ] + } + ] + end + + defp stop_event_children(false) do + Logger.info("#{__MODULE__} STOP_EVENTS_ENABLED=false") + [] + end + @doc false def source_url(mod) do case Application.get_env(:state_mediator, mod)[:source] do diff --git a/config/runtime.exs b/config/runtime.exs index 624b915b3..23b63e2de 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -82,6 +82,11 @@ if is_prod? and is_release? do s3_object: System.fetch_env!("CR_CROWDING_S3_OBJECT"), source: System.fetch_env!("CR_CROWDING_SOURCE") + config :state_mediator, :stop_events, + enabled: System.get_env("STOP_EVENTS_ENABLED", "false"), + s3_bucket: System.get_env("STOP_EVENTS_S3_BUCKET"), + s3_object: System.get_env("STOP_EVENTS_S3_OBJECT") + config :recaptcha, enabled: true, public_key: System.fetch_env!("RECAPTCHA_PUBLIC_KEY"), From b56f50aab83d7f77cb882fae64c07f1fd5d4a191 Mon Sep 17 00:00:00 2001 From: "Corey N. Runkel" Date: Wed, 25 Feb 2026 12:06:02 -0500 Subject: [PATCH 2/6] Clean up types and so forth --- .../controllers/stop_event_controller.ex | 81 +++-- .../stop_event_controller_test.exs | 287 ++++++++++++------ apps/model/lib/model/stop_event.ex | 4 +- apps/parse/lib/parse/stop_events.ex | 8 +- apps/parse/test/parse/stop_events_test.exs | 18 +- apps/state/lib/state/stop_event.ex | 84 +++-- apps/state/test/state/stop_event_test.exs | 50 +-- 7 files changed, 309 insertions(+), 223 deletions(-) diff --git a/apps/api_web/lib/api_web/controllers/stop_event_controller.ex b/apps/api_web/lib/api_web/controllers/stop_event_controller.ex index a9e2bf9f8..3fa94208c 100644 --- a/apps/api_web/lib/api_web/controllers/stop_event_controller.ex +++ b/apps/api_web/lib/api_web/controllers/stop_event_controller.ex @@ -25,7 +25,7 @@ defmodule ApiWeb.StopEventController do - The stop sequence number - Whether the trip was a revenue trip - Stop events are identified by a composite key of trip_id, route_id, vehicle_id, and stop_id. + Stop events are identified by a composite key of trip_id, route_id, vehicle_id, and current_stop_sequence. """ def state_module, do: State.StopEvent @@ -88,60 +88,41 @@ defmodule ApiWeb.StopEventController do def index_data(conn, params) do with :ok <- Params.validate_includes(params, @includes, conn), {:ok, filtered} <- Params.filter_params(params, @filters, conn) do - filtered - |> format_filters() - |> StopEvent.filter_by() - |> State.all(pagination_opts(params, conn)) + formatted_filters = format_filters(filtered) + + if map_size(formatted_filters) == 0 do + {:error, :filter_required} + else + formatted_filters + |> StopEvent.filter_by() + |> State.all(pagination_opts(params, conn)) + end else {:error, _, _} = error -> error end end @spec format_filters(%{optional(String.t()) => String.t()}) :: StopEvent.filters() - defp format_filters(filters, acc \\ %{}) + defp format_filters(filters) do + Enum.reduce(filters, %{}, fn + {"trip", trip_ids}, acc -> + Map.put(acc, :trip_ids, Params.split_on_comma(trip_ids)) - defp format_filters(%{"trip" => trip_ids} = filters, acc) do - new_acc = Map.put(acc, :trip_ids, Params.split_on_comma(trip_ids)) + {"stop", stop_ids}, acc -> + Map.put(acc, :stop_ids, Params.split_on_comma(stop_ids)) - filters - |> Map.delete("trip") - |> format_filters(new_acc) - end - - defp format_filters(%{"stop" => stop_ids} = filters, acc) do - new_acc = Map.put(acc, :stop_ids, Params.split_on_comma(stop_ids)) - - filters - |> Map.delete("stop") - |> format_filters(new_acc) - end + {"route", route_ids}, acc -> + Map.put(acc, :route_ids, Params.split_on_comma(route_ids)) - defp format_filters(%{"route" => route_ids} = filters, acc) do - new_acc = Map.put(acc, :route_ids, Params.split_on_comma(route_ids)) + {"vehicle", vehicle_ids}, acc -> + Map.put(acc, :vehicle_ids, Params.split_on_comma(vehicle_ids)) - filters - |> Map.delete("route") - |> format_filters(new_acc) - end - - defp format_filters(%{"vehicle" => vehicle_ids} = filters, acc) do - new_acc = Map.put(acc, :vehicle_ids, Params.split_on_comma(vehicle_ids)) + {"direction_id", direction_id}, acc -> + Map.put(acc, :direction_id, Params.direction_id(%{"direction_id" => direction_id})) - filters - |> Map.delete("vehicle") - |> format_filters(new_acc) - end - - defp format_filters(%{"direction_id" => direction_id} = filters, acc) do - new_acc = Map.put(acc, :direction_id, Params.direction_id(%{"direction_id" => direction_id})) - - filters - |> Map.delete("direction_id") - |> format_filters(new_acc) - end - - defp format_filters(_filters, acc) do - acc + _, acc -> + acc + end) end defp pagination_opts(params, conn) do @@ -167,7 +148,13 @@ defmodule ApiWeb.StopEventController do #{@description} """) - parameter(:id, :path, :string, "Unique identifier for stop event (trip_id-route_id-vehicle_id-stop_id)") + parameter( + :id, + :path, + :string, + "Unique identifier for stop event (trip_id-route_id-vehicle_id-current_stop_sequence)" + ) + include_parameters() consumes("application/vnd.api+json") @@ -293,7 +280,7 @@ defmodule ApiWeb.StopEventController do """ When the vehicle arrived at the stop, as seconds since Unix epoch (UTC). `null` if the first stop on the trip. """, - example: 1771966486, + example: 1_771_966_486, "x-nullable": true ) @@ -302,7 +289,7 @@ defmodule ApiWeb.StopEventController do """ When the vehicle departed from the stop, as seconds since Unix epoch (UTC). `null` if the last stop on the trip or if the vehicle has not yet departed. """, - example: 1771967246, + example: 1_771_967_246, "x-nullable": true ) end diff --git a/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs b/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs index 763fc3aca..a82be6149 100644 --- a/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs +++ b/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs @@ -9,10 +9,10 @@ defmodule ApiWeb.StopEventControllerTest do end describe "index_data/2" do - test "lists all entries on index", %{conn: conn} do + test "returns 400 with no filters", %{conn: conn} do State.StopEvent.new_state([ %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", start_date: ~D[2026-02-24], trip_id: "trip1", @@ -22,11 +22,11 @@ defmodule ApiWeb.StopEventControllerTest do revenue: :REVENUE, stop_id: "stop1", current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 }, %StopEvent{ - id: "trip2-route2-v2-stop2", + id: "trip2-route2-v2-1", vehicle_id: "v2", start_date: ~D[2026-02-24], trip_id: "trip2", @@ -36,23 +36,25 @@ defmodule ApiWeb.StopEventControllerTest do revenue: :NON_REVENUE, stop_id: "stop2", current_stop_sequence: 1, - arrived: 1771968343, + arrived: 1_771_968_343, departed: nil } ]) conn = get(conn, stop_event_path(conn, :index)) - response = json_response(conn, 200) - assert [ - %{"type" => "stop_event"}, - %{"type" => "stop_event"} - ] = response["data"] + assert json_response(conn, 400)["errors"] == [ + %{ + "status" => "400", + "code" => "bad_request", + "detail" => "At least one filter[] is required." + } + ] end test "conforms to swagger response", %{swagger_schema: schema, conn: conn} do stop_event = %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", start_date: ~D[2026-02-24], trip_id: "trip1", @@ -62,20 +64,20 @@ defmodule ApiWeb.StopEventControllerTest do revenue: :REVENUE, stop_id: "stop1", current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 } State.StopEvent.new_state([stop_event]) - response = get(conn, stop_event_path(conn, :index)) + response = get(conn, stop_event_path(conn, :index, %{"filter" => %{"trip" => "trip1"}})) assert validate_resp_schema(response, schema, "StopEvents") end test "can filter by trip", %{conn: conn} do State.StopEvent.new_state([ %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -85,11 +87,12 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 }, %StopEvent{ - id: "trip2-route2-stop2", + id: "trip2-route2-v2-1", + vehicle_id: "v2", trip_id: "trip2", direction_id: 0, route_id: "route2", @@ -98,20 +101,20 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "11:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771968343, + arrived: 1_771_968_343, departed: nil } ]) conn = get(conn, stop_event_path(conn, :index, %{"filter" => %{"trip" => "trip1"}})) - assert [%{"id" => "trip1-route1-v1-stop1"}] = json_response(conn, 200)["data"] + assert [%{"id" => "trip1-route1-v1-1"}] = json_response(conn, 200)["data"] end test "can filter by stop", %{conn: conn} do State.StopEvent.new_state([ %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -121,11 +124,11 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 }, %StopEvent{ - id: "trip1-route1-v1-stop2", + id: "trip1-route1-v1-2", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -135,20 +138,20 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 2, - arrived: 1771967286, - departed: 1771967333 + arrived: 1_771_967_286, + departed: 1_771_967_333 } ]) conn = get(conn, stop_event_path(conn, :index, %{"filter" => %{"stop" => "stop2"}})) - assert [%{"id" => "trip1-route1-v1-stop2"}] = json_response(conn, 200)["data"] + assert [%{"id" => "trip1-route1-v1-2"}] = json_response(conn, 200)["data"] end test "can filter by route", %{conn: conn} do State.StopEvent.new_state([ %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -158,11 +161,12 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 }, %StopEvent{ - id: "trip2-route2-stop2", + id: "trip2-route2-v2-1", + vehicle_id: "v2", trip_id: "trip2", direction_id: 0, route_id: "route2", @@ -171,20 +175,20 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "11:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771968343, + arrived: 1_771_968_343, departed: nil } ]) conn = get(conn, stop_event_path(conn, :index, %{"filter" => %{"route" => "route1"}})) - assert [%{"id" => "trip1-route1-v1-stop1"}] = json_response(conn, 200)["data"] + assert [%{"id" => "trip1-route1-v1-1"}] = json_response(conn, 200)["data"] end test "can filter by direction_id", %{conn: conn} do State.StopEvent.new_state([ %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -194,11 +198,12 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 }, %StopEvent{ - id: "trip2-route2-stop2", + id: "trip2-route2-v2-1", + vehicle_id: "v2", trip_id: "trip2", direction_id: 1, route_id: "route2", @@ -207,7 +212,7 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "11:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771968343, + arrived: 1_771_968_343, departed: nil } ]) @@ -215,13 +220,93 @@ defmodule ApiWeb.StopEventControllerTest do conn = get(conn, stop_event_path(conn, :index, %{"filter" => %{"direction_id" => "0"}})) - assert [%{"id" => "trip1-route1-v1-stop1"}] = json_response(conn, 200)["data"] + assert [%{"id" => "trip1-route1-v1-1"}] = json_response(conn, 200)["data"] + end + + test "can filter by vehicle", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-1", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1_771_966_486, + departed: 1_771_967_246 + }, + %StopEvent{ + id: "trip2-route2-v2-1", + vehicle_id: "v2", + trip_id: "trip2", + direction_id: 0, + route_id: "route2", + stop_id: "stop2", + start_date: ~D[2026-02-24], + start_time: "11:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1_771_968_343, + departed: nil + } + ]) + + conn = get(conn, stop_event_path(conn, :index, %{"filter" => %{"vehicle" => "v1"}})) + + assert [%{"id" => "trip1-route1-v1-1"}] = json_response(conn, 200)["data"] + end + + test "can filter by vehicle and route simultaneously", %{conn: conn} do + State.StopEvent.new_state([ + %StopEvent{ + id: "trip1-route1-v1-1", + vehicle_id: "v1", + trip_id: "trip1", + direction_id: 0, + route_id: "route1", + stop_id: "stop1", + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1_771_966_486, + departed: 1_771_967_246 + }, + %StopEvent{ + id: "trip2-route1-v2-1", + vehicle_id: "v2", + trip_id: "trip2", + direction_id: 0, + route_id: "route1", + stop_id: "stop2", + start_date: ~D[2026-02-24], + start_time: "11:00:00", + revenue: :REVENUE, + current_stop_sequence: 1, + arrived: 1_771_968_343, + departed: nil + } + ]) + + conn = + get( + conn, + stop_event_path(conn, :index, %{ + "filter" => %{"vehicle" => "v1", "route" => "route1"} + }) + ) + + assert [%{"id" => "trip1-route1-v1-1"}] = json_response(conn, 200)["data"] end test "can filter by route and direction_id simultaneously", %{conn: conn} do State.StopEvent.new_state([ %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -231,11 +316,11 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 }, %StopEvent{ - id: "trip2-route1-v2-stop2", + id: "trip2-route1-v2-2", vehicle_id: "v2", trip_id: "trip2", direction_id: 1, @@ -245,11 +330,11 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "11:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771968343, + arrived: 1_771_968_343, departed: nil }, %StopEvent{ - id: "trip3-route2-v3-stop3", + id: "trip3-route2-v3-3", vehicle_id: "v3", trip_id: "trip3", direction_id: 0, @@ -259,8 +344,8 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "12:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771969000, - departed: 1771969100 + arrived: 1_771_969_000, + departed: 1_771_969_100 } ]) @@ -272,13 +357,13 @@ defmodule ApiWeb.StopEventControllerTest do }) ) - assert [%{"id" => "trip1-route1-v1-stop1"}] = json_response(conn, 200)["data"] + assert [%{"id" => "trip1-route1-v1-1"}] = json_response(conn, 200)["data"] end test "can filter by trip and stop simultaneously", %{conn: conn} do State.StopEvent.new_state([ %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -288,11 +373,11 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 }, %StopEvent{ - id: "trip1-route1-v1-stop2", + id: "trip1-route1-v1-2", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -302,11 +387,11 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 2, - arrived: 1771967286, - departed: 1771967333 + arrived: 1_771_967_286, + departed: 1_771_967_333 }, %StopEvent{ - id: "trip2-route1-v2-stop2", + id: "trip2-route1-v2-2", vehicle_id: "v2", trip_id: "trip2", direction_id: 0, @@ -316,7 +401,7 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "11:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771968343, + arrived: 1_771_968_343, departed: nil } ]) @@ -327,13 +412,13 @@ defmodule ApiWeb.StopEventControllerTest do stop_event_path(conn, :index, %{"filter" => %{"trip" => "trip1", "stop" => "stop2"}}) ) - assert [%{"id" => "trip1-route1-v1-stop2"}] = json_response(conn, 200)["data"] + assert [%{"id" => "trip1-route1-v1-2"}] = json_response(conn, 200)["data"] end test "can filter by route, stop, and direction_id simultaneously", %{conn: conn} do State.StopEvent.new_state([ %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -343,11 +428,11 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 }, %StopEvent{ - id: "trip2-route1-stop1", + id: "trip2-route1-1", trip_id: "trip2", direction_id: 1, route_id: "route1", @@ -356,11 +441,11 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "11:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771968343, + arrived: 1_771_968_343, departed: nil }, %StopEvent{ - id: "trip3-route1-v3-stop2", + id: "trip3-route1-v3-2", vehicle_id: "v3", trip_id: "trip3", direction_id: 0, @@ -370,11 +455,11 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "12:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771969000, - departed: 1771969100 + arrived: 1_771_969_000, + departed: 1_771_969_100 }, %StopEvent{ - id: "trip4-route2-v4-stop1", + id: "trip4-route2-v4-1", vehicle_id: "v4", trip_id: "trip4", direction_id: 0, @@ -384,8 +469,8 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "13:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771970000, - departed: 1771970200 + arrived: 1_771_970_000, + departed: 1_771_970_200 } ]) @@ -397,13 +482,13 @@ defmodule ApiWeb.StopEventControllerTest do }) ) - assert [%{"id" => "trip1-route1-v1-stop1"}] = json_response(conn, 200)["data"] + assert [%{"id" => "trip1-route1-v1-1"}] = json_response(conn, 200)["data"] end test "can filter by multiple trips, routes, and stops simultaneously", %{conn: conn} do State.StopEvent.new_state([ %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -413,11 +498,11 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 }, %StopEvent{ - id: "trip2-route1-v2-stop2", + id: "trip2-route1-v2-2", vehicle_id: "v2", trip_id: "trip2", direction_id: 0, @@ -427,11 +512,11 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "11:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771968343, + arrived: 1_771_968_343, departed: nil }, %StopEvent{ - id: "trip3-route2-v3-stop3", + id: "trip3-route2-v3-3", vehicle_id: "v3", trip_id: "trip3", direction_id: 0, @@ -441,11 +526,11 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "12:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771969000, - departed: 1771969100 + arrived: 1_771_969_000, + departed: 1_771_969_100 }, %StopEvent{ - id: "trip2-route2-v2-stop1", + id: "trip2-route2-v2-1", vehicle_id: "v2", trip_id: "trip2", direction_id: 1, @@ -455,8 +540,8 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "13:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771970000, - departed: 1771970200 + arrived: 1_771_970_000, + departed: 1_771_970_200 } ]) @@ -471,13 +556,13 @@ defmodule ApiWeb.StopEventControllerTest do response = json_response(conn, 200)["data"] ids = Enum.map(response, & &1["id"]) |> Enum.sort() # Both trip1-route1-stop1 and trip2-route2-stop1 match the filters - assert ids == ["trip1-route1-v1-stop1", "trip2-route2-v2-stop1"] + assert ids == ["trip1-route1-v1-1", "trip2-route2-v2-1"] end test "returns empty when filters match no records", %{conn: conn} do State.StopEvent.new_state([ %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -487,8 +572,8 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 } ]) @@ -506,7 +591,7 @@ defmodule ApiWeb.StopEventControllerTest do test "pagination works", %{conn: conn} do State.StopEvent.new_state([ %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -516,11 +601,11 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 }, %StopEvent{ - id: "trip1-route1-v1-stop2", + id: "trip1-route1-v1-2", vehicle_id: "v1", trip_id: "trip1", direction_id: 0, @@ -530,13 +615,19 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, current_stop_sequence: 2, - arrived: 1771967286, - departed: 1771967333 + arrived: 1_771_967_286, + departed: 1_771_967_333 } ]) conn = - get(conn, stop_event_path(conn, :index, %{"page" => %{"limit" => "1", "offset" => "0"}})) + get( + conn, + stop_event_path(conn, :index, %{ + "filter" => %{"trip" => "trip1"}, + "page" => %{"limit" => "1", "offset" => "0"} + }) + ) response = json_response(conn, 200) assert length(response["data"]) == 1 @@ -547,7 +638,7 @@ defmodule ApiWeb.StopEventControllerTest do describe "show_data/2" do test "shows chosen resource", %{conn: conn} do stop_event = %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", start_date: ~D[2026-02-24], trip_id: "trip1", @@ -557,8 +648,8 @@ defmodule ApiWeb.StopEventControllerTest do revenue: :REVENUE, stop_id: "stop1", current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 } State.StopEvent.new_state([stop_event]) @@ -569,7 +660,7 @@ defmodule ApiWeb.StopEventControllerTest do test "conforms to swagger response", %{swagger_schema: schema, conn: conn} do stop_event = %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", start_date: ~D[2026-02-24], trip_id: "trip1", @@ -579,8 +670,8 @@ defmodule ApiWeb.StopEventControllerTest do revenue: :REVENUE, stop_id: "stop1", current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 } State.StopEvent.new_state([stop_event]) diff --git a/apps/model/lib/model/stop_event.ex b/apps/model/lib/model/stop_event.ex index 7ed43e48d..6125b2237 100644 --- a/apps/model/lib/model/stop_event.ex +++ b/apps/model/lib/model/stop_event.ex @@ -23,7 +23,7 @@ defmodule Model.StopEvent do ] @typedoc """ - * `:id` - Composite key: `{trip_id}-{route_id}-{vehicle_id}-{stop_id}`. + * `:id` - Composite key: `{trip_id}-{route_id}-{vehicle_id}-{current_stop_sequence}`. * `:vehicle_id` - The vehicle serving this trip. See [GTFS Realtime `FeedMessage` `FeedEntity` `VehiclePosition` `VehicleDescriptor` `id`](https://github.com/google/transit/blob/master/gtfs-realtime/spec/en/reference.md#message-vehicledescriptor). * `:start_date` - The service date of the `trip_id`. @@ -48,7 +48,7 @@ defmodule Model.StopEvent do trip_id: Model.Trip.id(), direction_id: Model.Direction.id(), route_id: Model.Route.id(), - start_time: non_neg_integer, + start_time: String.t(), stop_id: Model.Stop.id(), current_stop_sequence: non_neg_integer, revenue: :REVENUE | :NON_REVENUE, diff --git a/apps/parse/lib/parse/stop_events.ex b/apps/parse/lib/parse/stop_events.ex index dfc41c626..12f1c5db9 100644 --- a/apps/parse/lib/parse/stop_events.ex +++ b/apps/parse/lib/parse/stop_events.ex @@ -70,11 +70,9 @@ defmodule Parse.StopEvents do }, trip_data ) do - id = "#{trip_data.trip_id}-#{trip_data.route_id}-#{trip_data.vehicle_id}-#{stop_id}" - [ %Model.StopEvent{ - id: id, + id: build_composite_key(trip_data, current_stop_sequence), vehicle_id: trip_data.vehicle_id, start_date: trip_data.start_date, trip_id: trip_data.trip_id, @@ -95,6 +93,10 @@ defmodule Parse.StopEvents do [] end + defp build_composite_key(trip_data, current_stop_sequence) do + "#{trip_data.trip_id}-#{trip_data.route_id}-#{trip_data.vehicle_id}-#{current_stop_sequence}" + end + defp parse_date(<>) do case Date.new(String.to_integer(year), String.to_integer(month), String.to_integer(day)) do {:ok, date} -> {:ok, date} diff --git a/apps/parse/test/parse/stop_events_test.exs b/apps/parse/test/parse/stop_events_test.exs index 142257a1d..f5f1684a0 100644 --- a/apps/parse/test/parse/stop_events_test.exs +++ b/apps/parse/test/parse/stop_events_test.exs @@ -17,7 +17,7 @@ defmodule Parse.StopEventsTest do assert length(result) == 3 assert %StopEvent{ - id: "73885810-64-y2071-2231", + id: "73885810-64-y2071-1", vehicle_id: "y2071", start_date: ~D[2026-02-24], trip_id: "73885810", @@ -27,12 +27,12 @@ defmodule Parse.StopEventsTest do revenue: :REVENUE, stop_id: "2231", current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 } in result assert %StopEvent{ - id: "73885810-64-y2071-12232", + id: "73885810-64-y2071-2", vehicle_id: "y2071", start_date: ~D[2026-02-24], trip_id: "73885810", @@ -42,12 +42,12 @@ defmodule Parse.StopEventsTest do revenue: :REVENUE, stop_id: "12232", current_stop_sequence: 2, - arrived: 1771967286, - departed: 1771967333 + arrived: 1_771_967_286, + departed: 1_771_967_333 } in result assert %StopEvent{ - id: "73221192-Green-E-G-10077-70512", + id: "73221192-Green-E-G-10077-4", vehicle_id: "G-10077", start_date: ~D[2026-02-24], trip_id: "73221192", @@ -57,8 +57,8 @@ defmodule Parse.StopEventsTest do revenue: :REVENUE, stop_id: "70512", current_stop_sequence: 4, - arrived: 1771946303, - departed: 1771946479 + arrived: 1_771_946_303, + departed: 1_771_946_479 } in result end diff --git a/apps/state/lib/state/stop_event.ex b/apps/state/lib/state/stop_event.ex index 7559c61d7..f9a33dc1d 100644 --- a/apps/state/lib/state/stop_event.ex +++ b/apps/state/lib/state/stop_event.ex @@ -32,31 +32,31 @@ defmodule State.StopEvent do def filter_by(%{trip_ids: trip_ids} = filters) when is_list(trip_ids) and trip_ids != [] do trip_ids |> by_trip_ids() - |> apply_additional_filters(Map.delete(filters, :trip_ids)) + |> apply_additional_filters(filters) end def filter_by(%{stop_ids: stop_ids} = filters) when is_list(stop_ids) and stop_ids != [] do stop_ids |> by_stop_ids() - |> apply_additional_filters(Map.delete(filters, :stop_ids)) + |> apply_additional_filters(filters) end def filter_by(%{route_ids: route_ids} = filters) when is_list(route_ids) and route_ids != [] do route_ids |> by_route_ids() - |> apply_additional_filters(Map.delete(filters, :route_ids)) + |> apply_additional_filters(filters) end def filter_by(%{vehicle_ids: vehicle_ids} = filters) when is_list(vehicle_ids) and vehicle_ids != [] do vehicle_ids |> by_vehicle_ids() - |> apply_additional_filters(Map.delete(filters, :vehicle_ids)) + |> apply_additional_filters(filters) end - def filter_by(%{direction_id: direction_id}) do + def filter_by(%{direction_id: _direction_id} = filters) do all() - |> Enum.filter(fn %StopEvent{direction_id: d_id} -> d_id == direction_id end) + |> apply_additional_filters(filters) end def filter_by(%{} = map) when map_size(map) == 0 do @@ -67,54 +67,44 @@ defmodule State.StopEvent do [] end - defp apply_additional_filters(events, %{trip_ids: trip_ids} = filters) - when is_list(trip_ids) do - trip_id_set = MapSet.new(trip_ids) - - Enum.filter(events, fn %StopEvent{trip_id: trip_id} -> - MapSet.member?(trip_id_set, trip_id) + # Pre-compute all MapSets once to avoid repeated creation during filtering + defp apply_additional_filters(events, filters) do + # Build all filter sets upfront + filter_sets = %{ + trip_ids: build_filter_set(filters[:trip_ids]), + stop_ids: build_filter_set(filters[:stop_ids]), + route_ids: build_filter_set(filters[:route_ids]), + vehicle_ids: build_filter_set(filters[:vehicle_ids]), + direction_id: filters[:direction_id] + } + + Enum.filter(events, fn event -> + matches_trip?(event, filter_sets.trip_ids) and + matches_stop?(event, filter_sets.stop_ids) and + matches_route?(event, filter_sets.route_ids) and + matches_vehicle?(event, filter_sets.vehicle_ids) and + matches_direction?(event, filter_sets.direction_id) end) - |> apply_additional_filters(Map.delete(filters, :trip_ids)) end - defp apply_additional_filters(events, %{stop_ids: stop_ids} = filters) - when is_list(stop_ids) do - stop_id_set = MapSet.new(stop_ids) + defp build_filter_set(nil), do: nil + defp build_filter_set([]), do: nil + defp build_filter_set(list) when is_list(list), do: MapSet.new(list) - Enum.filter(events, fn %StopEvent{stop_id: stop_id} -> - MapSet.member?(stop_id_set, stop_id) - end) - |> apply_additional_filters(Map.delete(filters, :stop_ids)) - end + defp matches_trip?(_event, nil), do: true + defp matches_trip?(%StopEvent{trip_id: trip_id}, set), do: MapSet.member?(set, trip_id) - defp apply_additional_filters(events, %{route_ids: route_ids} = filters) - when is_list(route_ids) do - route_id_set = MapSet.new(route_ids) + defp matches_stop?(_event, nil), do: true + defp matches_stop?(%StopEvent{stop_id: stop_id}, set), do: MapSet.member?(set, stop_id) - Enum.filter(events, fn %StopEvent{route_id: route_id} -> - MapSet.member?(route_id_set, route_id) - end) - |> apply_additional_filters(Map.delete(filters, :route_ids)) - end + defp matches_route?(_event, nil), do: true + defp matches_route?(%StopEvent{route_id: route_id}, set), do: MapSet.member?(set, route_id) - defp apply_additional_filters(events, %{vehicle_ids: vehicle_ids} = filters) - when is_list(vehicle_ids) do - vehicle_id_set = MapSet.new(vehicle_ids) + defp matches_vehicle?(_event, nil), do: true - Enum.filter(events, fn %StopEvent{vehicle_id: vehicle_id} -> - MapSet.member?(vehicle_id_set, vehicle_id) - end) - |> apply_additional_filters(Map.delete(filters, :vehicle_ids)) - end + defp matches_vehicle?(%StopEvent{vehicle_id: vehicle_id}, set), + do: MapSet.member?(set, vehicle_id) - defp apply_additional_filters(events, %{direction_id: direction_id} = filters) do - Enum.filter(events, fn %StopEvent{direction_id: d_id} -> - d_id == direction_id - end) - |> apply_additional_filters(Map.delete(filters, :direction_id)) - end - - defp apply_additional_filters(events, _filters) do - events - end + defp matches_direction?(_event, nil), do: true + defp matches_direction?(%StopEvent{direction_id: d_id}, direction_id), do: d_id == direction_id end diff --git a/apps/state/test/state/stop_event_test.exs b/apps/state/test/state/stop_event_test.exs index 0a07574f4..38baece79 100644 --- a/apps/state/test/state/stop_event_test.exs +++ b/apps/state/test/state/stop_event_test.exs @@ -7,7 +7,7 @@ defmodule State.StopEventTest do describe "filter_by/1" do setup do stop_event1 = %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", start_date: ~D[2026-02-24], trip_id: "trip1", @@ -17,12 +17,12 @@ defmodule State.StopEventTest do revenue: :REVENUE, stop_id: "stop1", current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 } stop_event2 = %StopEvent{ - id: "trip1-route1-v1-stop2", + id: "trip1-route1-v1-2", vehicle_id: "v1", start_date: ~D[2026-02-24], trip_id: "trip1", @@ -32,12 +32,12 @@ defmodule State.StopEventTest do revenue: :REVENUE, stop_id: "stop2", current_stop_sequence: 2, - arrived: 1771967286, - departed: 1771967333 + arrived: 1_771_967_286, + departed: 1_771_967_333 } stop_event3 = %StopEvent{ - id: "trip2-route2-v2-stop3", + id: "trip2-route2-v2-3", vehicle_id: "v2", start_date: ~D[2026-02-24], trip_id: "trip2", @@ -47,7 +47,7 @@ defmodule State.StopEventTest do revenue: :NON_REVENUE, stop_id: "stop3", current_stop_sequence: 1, - arrived: 1771968343, + arrived: 1_771_968_343, departed: nil } @@ -80,7 +80,7 @@ defmodule State.StopEventTest do assert e3 in result end - test "filters by stop_id", %{event1: e1, event2: e2} do + test "filters by stop_id", %{event1: e1, event2: _e2} do result = filter_by(%{stop_ids: ["stop1"]}) assert result == [e1] end @@ -108,6 +108,22 @@ defmodule State.StopEventTest do assert e3 in result end + test "filters by vehicle_id", %{event1: e1, event2: e2, event3: e3} do + result = filter_by(%{vehicle_ids: ["v1"]}) + assert length(result) == 2 + assert e1 in result + assert e2 in result + refute e3 in result + end + + test "filters by multiple vehicle_ids", %{event1: e1, event2: e2, event3: e3} do + result = filter_by(%{vehicle_ids: ["v1", "v2"]}) + assert length(result) == 3 + assert e1 in result + assert e2 in result + assert e3 in result + end + test "filters by direction_id", %{event1: e1, event2: e2, event3: e3} do result = filter_by(%{direction_id: 0}) assert length(result) == 2 @@ -144,7 +160,7 @@ defmodule State.StopEventTest do test "filters by multiple values across all filter types" do # Add more test data for this test stop_event4 = %StopEvent{ - id: "trip2-route1-v2-stop1", + id: "trip2-route1-v2-1", vehicle_id: "v2", start_date: ~D[2026-02-24], trip_id: "trip2", @@ -154,8 +170,8 @@ defmodule State.StopEventTest do revenue: :REVENUE, stop_id: "stop1", current_stop_sequence: 1, - arrived: 1771969000, - departed: 1771969100 + arrived: 1_771_969_000, + departed: 1_771_969_100 } all_events = State.StopEvent.all() @@ -178,7 +194,7 @@ defmodule State.StopEventTest do assert Enum.all?(result, fn e -> e.trip_id in ["trip1", "trip2"] end) end - test "returns empty when combining filters that match no records", %{event1: e1, event2: e2} do + test "returns empty when combining filters that match no records", %{event1: _e1, event2: _e2} do # event1 and event2 both have route1, but only event1 has stop1 # Filtering for route1, stop2, and direction_id 1 should return nothing result = filter_by(%{route_ids: ["route1"], stop_ids: ["stop2"], direction_id: 1}) @@ -202,7 +218,7 @@ defmodule State.StopEventTest do describe "by_id/1" do test "returns stop event by id" do stop_event = %StopEvent{ - id: "trip1-route1-v1-stop1", + id: "trip1-route1-v1-1", vehicle_id: "v1", start_date: ~D[2026-02-24], trip_id: "trip1", @@ -212,13 +228,13 @@ defmodule State.StopEventTest do revenue: :REVENUE, stop_id: "stop1", current_stop_sequence: 1, - arrived: 1771966486, - departed: 1771967246 + arrived: 1_771_966_486, + departed: 1_771_967_246 } State.StopEvent.new_state([stop_event]) - assert by_id("trip1-route1-v1-stop1") == stop_event + assert by_id("trip1-route1-v1-1") == stop_event end test "returns nil for non-existent id" do From e396338ee0e0fbe92dc82c04823a4cdd080985e6 Mon Sep 17 00:00:00 2001 From: "Corey N. Runkel" Date: Sun, 1 Mar 2026 14:48:00 -0500 Subject: [PATCH 3/6] Better filtering --- .../controllers/stop_event_controller.ex | 6 +- apps/api_web/lib/api_web/router.ex | 2 +- .../lib/api_web/views/stop_event_view.ex | 2 +- .../stop_event_controller_test.exs | 68 +++---- apps/model/lib/model/stop_event.ex | 12 +- apps/parse/lib/parse/stop_events.ex | 21 +- apps/parse/test/parse/stop_events_test.exs | 34 ++-- apps/state/lib/state/stop_event.ex | 192 ++++++++++++------ apps/state/test/state/stop_event_test.exs | 180 +++++++++++++++- apps/state_mediator/lib/state_mediator.ex | 2 +- 10 files changed, 387 insertions(+), 132 deletions(-) diff --git a/apps/api_web/lib/api_web/controllers/stop_event_controller.ex b/apps/api_web/lib/api_web/controllers/stop_event_controller.ex index 3fa94208c..e5ad7f7f5 100644 --- a/apps/api_web/lib/api_web/controllers/stop_event_controller.ex +++ b/apps/api_web/lib/api_web/controllers/stop_event_controller.ex @@ -25,7 +25,7 @@ defmodule ApiWeb.StopEventController do - The stop sequence number - Whether the trip was a revenue trip - Stop events are identified by a composite key of trip_id, route_id, vehicle_id, and current_stop_sequence. + Stop events are identified by a composite key of trip_id, route_id, vehicle_id, and stop_sequence. """ def state_module, do: State.StopEvent @@ -152,7 +152,7 @@ defmodule ApiWeb.StopEventController do :id, :path, :string, - "Unique identifier for stop event (trip_id-route_id-vehicle_id-current_stop_sequence)" + "Unique identifier for stop event (trip_id-route_id-vehicle_id-stop_sequence)" ) include_parameters() @@ -267,7 +267,7 @@ defmodule ApiWeb.StopEventController do example: "2231" ) - current_stop_sequence( + stop_sequence( :integer, """ The stop sequence number along the trip. Increases monotonically but values need not be consecutive. diff --git a/apps/api_web/lib/api_web/router.ex b/apps/api_web/lib/api_web/router.ex index 96f404e23..d8be1b907 100644 --- a/apps/api_web/lib/api_web/router.ex +++ b/apps/api_web/lib/api_web/router.ex @@ -100,8 +100,8 @@ defmodule ApiWeb.Router do resources("/live_facilities", LiveFacilityController, only: [:index, :show]) resources("/live-facilities", LiveFacilityController, only: [:index, :show]) resources("/services", ServiceController, only: [:index, :show]) - resources("/stop-events", StopEventController, only: [:index, :show]) resources("/stop_events", StopEventController, only: [:index, :show]) + resources("/stop-events", StopEventController, only: [:index, :show]) end scope "/docs/swagger" do diff --git a/apps/api_web/lib/api_web/views/stop_event_view.ex b/apps/api_web/lib/api_web/views/stop_event_view.ex index 8cfc0ba6e..00d896358 100644 --- a/apps/api_web/lib/api_web/views/stop_event_view.ex +++ b/apps/api_web/lib/api_web/views/stop_event_view.ex @@ -43,7 +43,7 @@ defmodule ApiWeb.StopEventView do :start_time, :revenue, :stop_id, - :current_stop_sequence, + :stop_sequence, :arrived, :departed ]) diff --git a/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs b/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs index a82be6149..90725fa0a 100644 --- a/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs +++ b/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs @@ -21,7 +21,7 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, stop_id: "stop1", - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 }, @@ -35,7 +35,7 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "11:00:00", revenue: :NON_REVENUE, stop_id: "stop2", - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_968_343, departed: nil } @@ -63,7 +63,7 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, stop_id: "stop1", - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 } @@ -86,7 +86,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 }, @@ -100,7 +100,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "11:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_968_343, departed: nil } @@ -123,7 +123,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 }, @@ -137,7 +137,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 2, + stop_sequence: 2, arrived: 1_771_967_286, departed: 1_771_967_333 } @@ -160,7 +160,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 }, @@ -174,7 +174,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "11:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_968_343, departed: nil } @@ -197,7 +197,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 }, @@ -211,7 +211,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "11:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_968_343, departed: nil } @@ -235,7 +235,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 }, @@ -249,7 +249,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "11:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_968_343, departed: nil } @@ -272,7 +272,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 }, @@ -286,7 +286,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "11:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_968_343, departed: nil } @@ -315,7 +315,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 }, @@ -329,7 +329,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "11:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_968_343, departed: nil }, @@ -343,7 +343,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "12:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_969_000, departed: 1_771_969_100 } @@ -372,7 +372,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 }, @@ -386,7 +386,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 2, + stop_sequence: 2, arrived: 1_771_967_286, departed: 1_771_967_333 }, @@ -400,7 +400,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "11:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_968_343, departed: nil } @@ -427,7 +427,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 }, @@ -440,7 +440,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "11:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_968_343, departed: nil }, @@ -454,7 +454,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "12:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_969_000, departed: 1_771_969_100 }, @@ -468,7 +468,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "13:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_970_000, departed: 1_771_970_200 } @@ -497,7 +497,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 }, @@ -511,7 +511,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "11:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_968_343, departed: nil }, @@ -525,7 +525,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "12:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_969_000, departed: 1_771_969_100 }, @@ -539,7 +539,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "13:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_970_000, departed: 1_771_970_200 } @@ -571,7 +571,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 } @@ -600,7 +600,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 }, @@ -614,7 +614,7 @@ defmodule ApiWeb.StopEventControllerTest do start_date: ~D[2026-02-24], start_time: "10:00:00", revenue: :REVENUE, - current_stop_sequence: 2, + stop_sequence: 2, arrived: 1_771_967_286, departed: 1_771_967_333 } @@ -647,7 +647,7 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, stop_id: "stop1", - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 } @@ -669,7 +669,7 @@ defmodule ApiWeb.StopEventControllerTest do start_time: "10:00:00", revenue: :REVENUE, stop_id: "stop1", - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 } diff --git a/apps/model/lib/model/stop_event.ex b/apps/model/lib/model/stop_event.ex index 6125b2237..ec7177822 100644 --- a/apps/model/lib/model/stop_event.ex +++ b/apps/model/lib/model/stop_event.ex @@ -17,13 +17,13 @@ defmodule Model.StopEvent do :start_time, :revenue, :stop_id, - :current_stop_sequence, + :stop_sequence, :arrived, :departed ] @typedoc """ - * `:id` - Composite key: `{trip_id}-{route_id}-{vehicle_id}-{current_stop_sequence}`. + * `:id` - Composite key: `{trip_id}-{route_id}-{vehicle_id}-{stop_sequence}`. * `:vehicle_id` - The vehicle serving this trip. See [GTFS Realtime `FeedMessage` `FeedEntity` `VehiclePosition` `VehicleDescriptor` `id`](https://github.com/google/transit/blob/master/gtfs-realtime/spec/en/reference.md#message-vehicledescriptor). * `:start_date` - The service date of the `trip_id`. @@ -36,21 +36,21 @@ defmodule Model.StopEvent do * `:revenue` - Whether or not the stop event is for a revenue trip. * `:stop_id` - Stop associated with arrived/departed. See [GTFS Realtime `FeedMesage` `FeedEntity` `TripUpdate` `StopTimeUpdate` `stop_id`](https://github.com/google/transit/blob/master/gtfs-realtime/spec/en/reference.md#message-stoptimeupdate). - * `:current_stop_sequence` - The sequence of the stop along the `trip_id`. The stop sequence increases monotonically but values need not be consecutive. - See [GTFS Realtime `FeedMesage` `FeedEntity` `VehiclePosition` `current_stop_sequence`](https://github.com/google/transit/blob/master/gtfs-realtime/spec/en/reference.md#message-vehicleposition). + * `:stop_sequence` - The sequence of the stop along the `trip_id`. The stop sequence increases monotonically but values need not be consecutive. + See [GTFS `stop_times.txt` `stop_sequence`](https://github.com/google/transit/blob/master/gtfs/spec/en/reference.md#stop_timestxt). * `:arrived` - When the vehicle arrived at the stop as seconds since Unix epoch in timezone `America/New_York`. `nil` if the first stop (`stop_id`) on the `trip_id`. * `:departed` - When the vehicle arrived at the stop as seconds since Unix epoch in timezone `America/New_York`. `nil` if the last stop (`stop_id`) on the `trip_id` """ @type t :: %__MODULE__{ id: String.t(), - vehicle_id: String.t(), + vehicle_id: Model.Vehicle.id(), start_date: Date.t(), trip_id: Model.Trip.id(), direction_id: Model.Direction.id(), route_id: Model.Route.id(), start_time: String.t(), stop_id: Model.Stop.id(), - current_stop_sequence: non_neg_integer, + stop_sequence: non_neg_integer, revenue: :REVENUE | :NON_REVENUE, arrived: DateTime.t() | nil, departed: DateTime.t() | nil diff --git a/apps/parse/lib/parse/stop_events.ex b/apps/parse/lib/parse/stop_events.ex index 12f1c5db9..edf1a927c 100644 --- a/apps/parse/lib/parse/stop_events.ex +++ b/apps/parse/lib/parse/stop_events.ex @@ -1,6 +1,6 @@ defmodule Parse.StopEvents do @moduledoc """ - Parser for the Stop Events data from S3 (NDJSON format) + Parses stop_events new line-delimited JSON into a list of `%Model.StopEvent{}` structs. """ require Logger @@ -8,8 +8,8 @@ defmodule Parse.StopEvents do @behaviour Parse @impl Parse - def parse(binary) do - binary + def parse(body) do + body |> String.split("\n", trim: true) |> Enum.flat_map(&parse_line/1) end @@ -51,7 +51,10 @@ defmodule Parse.StopEvents do end) else error -> - Logger.warning("#{__MODULE__} parse_error error=#{inspect(error)} trip_id=#{trip_id}") + Logger.warning( + "#{__MODULE__} parse_error error=#{inspect(error)} trip_id=#{trip_id} vehicle_id=#{vehicle_id}" + ) + [] end end @@ -64,7 +67,7 @@ defmodule Parse.StopEvents do defp parse_stop_event( %{ "stop_id" => stop_id, - "current_stop_sequence" => current_stop_sequence, + "stop_sequence" => stop_sequence, "arrived" => arrived, "departed" => departed }, @@ -72,7 +75,7 @@ defmodule Parse.StopEvents do ) do [ %Model.StopEvent{ - id: build_composite_key(trip_data, current_stop_sequence), + id: build_composite_key(trip_data, stop_sequence), vehicle_id: trip_data.vehicle_id, start_date: trip_data.start_date, trip_id: trip_data.trip_id, @@ -81,7 +84,7 @@ defmodule Parse.StopEvents do start_time: trip_data.start_time, revenue: trip_data.revenue, stop_id: stop_id, - current_stop_sequence: current_stop_sequence, + stop_sequence: stop_sequence, arrived: arrived, departed: departed } @@ -93,8 +96,8 @@ defmodule Parse.StopEvents do [] end - defp build_composite_key(trip_data, current_stop_sequence) do - "#{trip_data.trip_id}-#{trip_data.route_id}-#{trip_data.vehicle_id}-#{current_stop_sequence}" + defp build_composite_key(trip_data, stop_sequence) do + "#{trip_data.trip_id}-#{trip_data.route_id}-#{trip_data.vehicle_id}-#{stop_sequence}" end defp parse_date(<>) do diff --git a/apps/parse/test/parse/stop_events_test.exs b/apps/parse/test/parse/stop_events_test.exs index f5f1684a0..d9ccfa3c7 100644 --- a/apps/parse/test/parse/stop_events_test.exs +++ b/apps/parse/test/parse/stop_events_test.exs @@ -8,8 +8,8 @@ defmodule Parse.StopEventsTest do describe "parse" do test "parses valid NDJSON data with multiple stop events" do ndjson = """ - {"id":"73885810-64-y2071","timestamp":1771968343,"start_date":"20260224","trip_id":"73885810","vehicle_id":"y2071","direction_id":0,"route_id":"64","start_time":"16:07:00","revenue":true,"stop_events":[{"stop_id":"2231","current_stop_sequence":1,"arrived":1771966486,"departed":1771967246},{"stop_id":"12232","current_stop_sequence":2,"arrived":1771967286,"departed":1771967333}]} - {"id":"73221192-Green-E-G-10077","timestamp":1771950045,"start_date":"20260224","trip_id":"73221192","vehicle_id":"G-10077","direction_id":0,"route_id":"Green-E","start_time":"10:16:00","revenue":true,"stop_events":[{"stop_id":"70512","current_stop_sequence":4,"arrived":1771946303,"departed":1771946479}]} + {"id":"73885810-64-y2071","timestamp":1771968343,"start_date":"20260224","trip_id":"73885810","vehicle_id":"y2071","direction_id":0,"route_id":"64","start_time":"16:07:00","revenue":true,"stop_events":[{"stop_id":"2231","stop_sequence":1,"arrived":1771966486,"departed":1771967246},{"stop_id":"12232","stop_sequence":2,"arrived":1771967286,"departed":1771967333}]} + {"id":"73221192-Green-E-G-10077","timestamp":1771950045,"start_date":"20260224","trip_id":"73221192","vehicle_id":"G-10077","direction_id":0,"route_id":"Green-E","start_time":"10:16:00","revenue":true,"stop_events":[{"stop_id":"70512","stop_sequence":4,"arrived":1771946303,"departed":1771946479}]} """ result = parse(ndjson) @@ -26,7 +26,7 @@ defmodule Parse.StopEventsTest do start_time: "16:07:00", revenue: :REVENUE, stop_id: "2231", - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 } in result @@ -41,7 +41,7 @@ defmodule Parse.StopEventsTest do start_time: "16:07:00", revenue: :REVENUE, stop_id: "12232", - current_stop_sequence: 2, + stop_sequence: 2, arrived: 1_771_967_286, departed: 1_771_967_333 } in result @@ -56,7 +56,7 @@ defmodule Parse.StopEventsTest do start_time: "10:16:00", revenue: :REVENUE, stop_id: "70512", - current_stop_sequence: 4, + stop_sequence: 4, arrived: 1_771_946_303, departed: 1_771_946_479 } in result @@ -64,7 +64,7 @@ defmodule Parse.StopEventsTest do test "handles null departed times for last stop" do ndjson = """ - {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","current_stop_sequence":1,"arrived":1771966486,"departed":null}]} + {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":1771966486,"departed":null}]} """ result = parse(ndjson) @@ -74,7 +74,7 @@ defmodule Parse.StopEventsTest do test "handles null arrived times for first stop" do ndjson = """ - {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","current_stop_sequence":1,"arrived":null,"departed":1771967246}]} + {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":null,"departed":1771967246}]} """ result = parse(ndjson) @@ -84,7 +84,7 @@ defmodule Parse.StopEventsTest do test "handles non-revenue trips" do ndjson = """ - {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":false,"stop_events":[{"stop_id":"stop1","current_stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":false,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} """ result = parse(ndjson) @@ -95,7 +95,7 @@ defmodule Parse.StopEventsTest do test "ignores empty lines in NDJSON" do ndjson = """ - {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","current_stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} """ @@ -106,15 +106,21 @@ defmodule Parse.StopEventsTest do test "logs and ignores lines with missing required fields" do ndjson = """ - {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","current_stop_sequence":1}]} - {"id":"valid","timestamp":1771968343,"start_date":"20260224","trip_id":"valid","vehicle_id":"v2","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop2","current_stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + {"id":"missing-times","timestamp":1771968343,"start_date":"20260224","trip_id":"missing","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1}]} + {"id":"valid-has-arrived","timestamp":1771968343,"start_date":"20260224","trip_id":"arrived","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":1771966486}]} + {"id":"valid-has-departed","timestamp":1771968343,"start_date":"20260224","trip_id":"departed","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"departed":1771967246}]} + {"id":"valid-has-both-times","timestamp":1771968343,"start_date":"20260224","trip_id":"both-times","vehicle_id":"v2","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop2","stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} """ log = capture_log(fn -> result = parse(ndjson) - assert length(result) == 1 - assert [%StopEvent{trip_id: "valid"}] = result + + assert [ + %StopEvent{trip_id: "both-times"}, + %StopEvent{trip_id: "arrived"}, + %StopEvent{trip_id: "departed"} + ] = result end) assert log =~ "missing_fields" @@ -122,7 +128,7 @@ defmodule Parse.StopEventsTest do test "logs and ignores lines with invalid date format" do ndjson = """ - {"id":"test-trip","timestamp":1771968343,"start_date":"invalid","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","current_stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + {"id":"test-trip","timestamp":1771968343,"start_date":"invalid","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} """ log = diff --git a/apps/state/lib/state/stop_event.ex b/apps/state/lib/state/stop_event.ex index f9a33dc1d..8f7ead7ea 100644 --- a/apps/state/lib/state/stop_event.ex +++ b/apps/state/lib/state/stop_event.ex @@ -11,15 +11,19 @@ defmodule State.StopEvent do alias Model.StopEvent alias Model.Stop alias Model.Trip + alias Model.Vehicle @type filters :: %{ optional(:trip_ids) => [Trip.id()], optional(:stop_ids) => [Stop.id()], optional(:route_ids) => [Route.id()], - optional(:vehicle_ids) => [String.t()], - optional(:direction_id) => 0 | 1 + optional(:vehicle_ids) => [Vehicle.id()], + optional(:direction_id) => Model.Direction.id() } + # Filter keys ordered by typical selectivity (most selective first) + @filter_keys [:trip_ids, :vehicle_ids, :stop_ids, :route_ids] + @spec by_id(String.t()) :: StopEvent.t() | nil def by_id(id) do case super(id) do @@ -28,83 +32,155 @@ defmodule State.StopEvent do end end - @spec filter_by(filters()) :: [StopEvent.t()] - def filter_by(%{trip_ids: trip_ids} = filters) when is_list(trip_ids) and trip_ids != [] do - trip_ids - |> by_trip_ids() - |> apply_additional_filters(filters) - end + @doc """ + Filters stop events based on the provided filter criteria. - def filter_by(%{stop_ids: stop_ids} = filters) when is_list(stop_ids) and stop_ids != [] do - stop_ids - |> by_stop_ids() - |> apply_additional_filters(filters) - end + At least one filter should be provided for efficient querying. The function + automatically selects the most selective index based on the number of values + in each filter list. - def filter_by(%{route_ids: route_ids} = filters) when is_list(route_ids) and route_ids != [] do - route_ids - |> by_route_ids() - |> apply_additional_filters(filters) - end + ## Options - def filter_by(%{vehicle_ids: vehicle_ids} = filters) - when is_list(vehicle_ids) and vehicle_ids != [] do - vehicle_ids - |> by_vehicle_ids() - |> apply_additional_filters(filters) - end + Accepts the same options as `State.all/2`: + * `:limit` - Maximum number of results to return + * `:offset` - Number of results to skip + * `:order_by` - Field(s) to sort by, e.g. `{:arrived, :asc}` + + ## Examples + + filter_by(%{trip_ids: ["trip1"]}) + filter_by(%{route_ids: ["Red"], direction_id: 0}, limit: 10) + + """ + @spec filter_by(filters(), Keyword.t()) :: + [StopEvent.t()] | {[StopEvent.t()], State.Pagination.Offsets.t()} + def filter_by(filters, opts \\ []) - def filter_by(%{direction_id: _direction_id} = filters) do + def filter_by(%{} = filters, opts) when map_size(filters) == 0 do all() - |> apply_additional_filters(filters) + |> State.all(opts) end - def filter_by(%{} = map) when map_size(map) == 0 do - all() + def filter_by(filters, opts) do + case select_best_index(filters) do + {:direction_id_only, direction_id} -> + # direction_id alone requires full scan + all() + |> filter_by_direction(direction_id) + |> State.all(opts) + + {:single_filter, filter_key, values} -> + # Single indexed filter - use index directly, no MapSet needed + values + |> fetch_by_index(filter_key) + |> maybe_filter_direction(filters[:direction_id]) + |> State.all(opts) + + {:multi_filter, primary_key, primary_values, remaining_filters} -> + # Multiple filters - use best index, then apply remaining with MapSets + primary_values + |> fetch_by_index(primary_key) + |> apply_additional_filters(remaining_filters) + |> State.all(opts) + + :no_filters -> + [] + end end - def filter_by(_filters) do - [] + # Selects the best index based on filter selectivity (smallest list first) + defp select_best_index(filters) do + indexed_filters = + @filter_keys + |> Enum.map(fn key -> {key, Map.get(filters, key)} end) + |> Enum.filter(fn {_key, values} -> is_list(values) and values != [] end) + |> Enum.sort_by(fn {_key, values} -> length(values) end) + + direction_id = filters[:direction_id] + + case {indexed_filters, direction_id} do + {[], nil} -> + :no_filters + + {[], direction_id} -> + {:direction_id_only, direction_id} + + {[{key, values}], nil} -> + {:single_filter, key, values} + + {[{key, values}], direction_id} -> + # Single indexed filter + direction_id + remaining = %{direction_id: direction_id} + {:multi_filter, key, values, remaining} + + {[{primary_key, primary_values} | rest], direction_id} -> + # Multiple indexed filters - build remaining filter map + remaining = + rest + |> Enum.into(%{}) + |> maybe_put_direction(direction_id) + + {:multi_filter, primary_key, primary_values, remaining} + end + end + + defp maybe_put_direction(map, nil), do: map + defp maybe_put_direction(map, direction_id), do: Map.put(map, :direction_id, direction_id) + + # Fetch records using the appropriate index + defp fetch_by_index(values, :trip_ids), do: by_trip_ids(values) + defp fetch_by_index(values, :stop_ids), do: by_stop_ids(values) + defp fetch_by_index(values, :route_ids), do: by_route_ids(values) + defp fetch_by_index(values, :vehicle_ids), do: by_vehicle_ids(values) + + # Simple direction filter for single-filter cases (no MapSet overhead) + defp filter_by_direction(events, direction_id) do + Enum.filter(events, fn %StopEvent{direction_id: d_id} -> d_id == direction_id end) end - # Pre-compute all MapSets once to avoid repeated creation during filtering + defp maybe_filter_direction(events, nil), do: events + defp maybe_filter_direction(events, direction_id), do: filter_by_direction(events, direction_id) + + # Apply additional filters using pre-computed MapSets (for multi-filter cases) + defp apply_additional_filters(events, filters) when map_size(filters) == 0, do: events + defp apply_additional_filters(events, filters) do - # Build all filter sets upfront - filter_sets = %{ - trip_ids: build_filter_set(filters[:trip_ids]), - stop_ids: build_filter_set(filters[:stop_ids]), - route_ids: build_filter_set(filters[:route_ids]), - vehicle_ids: build_filter_set(filters[:vehicle_ids]), - direction_id: filters[:direction_id] - } + # Build all filter sets upfront to avoid repeated MapSet creation + filter_specs = build_filter_specs(filters) Enum.filter(events, fn event -> - matches_trip?(event, filter_sets.trip_ids) and - matches_stop?(event, filter_sets.stop_ids) and - matches_route?(event, filter_sets.route_ids) and - matches_vehicle?(event, filter_sets.vehicle_ids) and - matches_direction?(event, filter_sets.direction_id) + Enum.all?(filter_specs, fn spec -> matches_filter?(event, spec) end) end) end - defp build_filter_set(nil), do: nil - defp build_filter_set([]), do: nil - defp build_filter_set(list) when is_list(list), do: MapSet.new(list) + # Build filter specifications with pre-computed MapSets + defp build_filter_specs(filters) do + [] + |> maybe_add_filter_spec(filters[:trip_ids], :trip_id) + |> maybe_add_filter_spec(filters[:stop_ids], :stop_id) + |> maybe_add_filter_spec(filters[:route_ids], :route_id) + |> maybe_add_filter_spec(filters[:vehicle_ids], :vehicle_id) + |> maybe_add_direction_spec(filters[:direction_id]) + end - defp matches_trip?(_event, nil), do: true - defp matches_trip?(%StopEvent{trip_id: trip_id}, set), do: MapSet.member?(set, trip_id) + defp maybe_add_filter_spec(specs, nil, _field), do: specs + defp maybe_add_filter_spec(specs, [], _field), do: specs - defp matches_stop?(_event, nil), do: true - defp matches_stop?(%StopEvent{stop_id: stop_id}, set), do: MapSet.member?(set, stop_id) + defp maybe_add_filter_spec(specs, values, field) when is_list(values) do + [{:set, field, MapSet.new(values)} | specs] + end - defp matches_route?(_event, nil), do: true - defp matches_route?(%StopEvent{route_id: route_id}, set), do: MapSet.member?(set, route_id) + defp maybe_add_direction_spec(specs, nil), do: specs - defp matches_vehicle?(_event, nil), do: true + defp maybe_add_direction_spec(specs, direction_id), + do: [{:eq, :direction_id, direction_id} | specs] - defp matches_vehicle?(%StopEvent{vehicle_id: vehicle_id}, set), - do: MapSet.member?(set, vehicle_id) + # Pattern match on filter specification type for efficient dispatch + defp matches_filter?(event, {:set, field, set}) do + MapSet.member?(set, Map.get(event, field)) + end - defp matches_direction?(_event, nil), do: true - defp matches_direction?(%StopEvent{direction_id: d_id}, direction_id), do: d_id == direction_id + defp matches_filter?(event, {:eq, field, value}) do + Map.get(event, field) == value + end end diff --git a/apps/state/test/state/stop_event_test.exs b/apps/state/test/state/stop_event_test.exs index 38baece79..f69ff15eb 100644 --- a/apps/state/test/state/stop_event_test.exs +++ b/apps/state/test/state/stop_event_test.exs @@ -16,7 +16,7 @@ defmodule State.StopEventTest do start_time: "10:00:00", revenue: :REVENUE, stop_id: "stop1", - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 } @@ -31,7 +31,7 @@ defmodule State.StopEventTest do start_time: "10:00:00", revenue: :REVENUE, stop_id: "stop2", - current_stop_sequence: 2, + stop_sequence: 2, arrived: 1_771_967_286, departed: 1_771_967_333 } @@ -46,7 +46,7 @@ defmodule State.StopEventTest do start_time: "11:00:00", revenue: :NON_REVENUE, stop_id: "stop3", - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_968_343, departed: nil } @@ -169,7 +169,7 @@ defmodule State.StopEventTest do start_time: "12:00:00", revenue: :REVENUE, stop_id: "stop1", - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_969_000, departed: 1_771_969_100 } @@ -215,6 +215,176 @@ defmodule State.StopEventTest do end end + describe "filter_by/2 with pagination" do + setup do + events = + for i <- 1..10 do + %StopEvent{ + id: "trip#{i}-route1-v1-#{i}", + vehicle_id: "v1", + start_date: ~D[2026-02-24], + trip_id: "trip#{i}", + direction_id: rem(i, 2), + route_id: "route1", + start_time: "10:00:00", + revenue: :REVENUE, + stop_id: "stop#{i}", + stop_sequence: i, + arrived: 1_771_966_486 + i * 100, + departed: 1_771_967_246 + i * 100 + } + end + + State.StopEvent.new_state(events) + {:ok, %{events: events}} + end + + test "supports limit option" do + {result, _pagination} = filter_by(%{route_ids: ["route1"]}, limit: 3) + assert length(result) == 3 + end + + test "supports offset option" do + all_results = filter_by(%{route_ids: ["route1"]}) + {offset_results, _pagination} = filter_by(%{route_ids: ["route1"]}, offset: 2, limit: 20) + + assert length(offset_results) == length(all_results) - 2 + end + + test "supports limit and offset together" do + {result, _pagination} = filter_by(%{route_ids: ["route1"]}, limit: 2, offset: 3) + assert length(result) == 2 + end + + test "supports order_by option" do + # Order by stop_sequence ascending + {result, _pagination} = + filter_by(%{route_ids: ["route1"]}, order_by: {:stop_sequence, :asc}, limit: 20) + + assert length(result) == 10 + assert hd(result).stop_sequence == 1 + assert List.last(result).stop_sequence == 10 + end + + test "combines pagination with filtering" do + # Filter by direction_id 0 (odd numbered trips: 1,3,5,7,9), limit 2 + {result, _pagination} = filter_by(%{direction_id: 0}, limit: 2) + assert length(result) == 2 + assert Enum.all?(result, fn e -> e.direction_id == 0 end) + end + end + + describe "filter_by/2 selectivity optimization" do + setup do + # Create data where vehicle_id is most selective (1 match), + # trip_id is medium (3 matches), route_id is least selective (5 matches) + events = [ + %StopEvent{ + id: "trip1-route1-v1-1", + vehicle_id: "v1", + trip_id: "trip1", + route_id: "route1", + stop_id: "stop1", + direction_id: 0, + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + stop_sequence: 1, + arrived: 1_771_966_486, + departed: 1_771_967_246 + }, + %StopEvent{ + id: "trip1-route1-v2-2", + vehicle_id: "v2", + trip_id: "trip1", + route_id: "route1", + stop_id: "stop2", + direction_id: 0, + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + stop_sequence: 2, + arrived: 1_771_966_586, + departed: 1_771_967_346 + }, + %StopEvent{ + id: "trip1-route1-v3-3", + vehicle_id: "v3", + trip_id: "trip1", + route_id: "route1", + stop_id: "stop3", + direction_id: 0, + start_date: ~D[2026-02-24], + start_time: "10:00:00", + revenue: :REVENUE, + stop_sequence: 3, + arrived: 1_771_966_686, + departed: 1_771_967_446 + }, + %StopEvent{ + id: "trip2-route1-v4-1", + vehicle_id: "v4", + trip_id: "trip2", + route_id: "route1", + stop_id: "stop1", + direction_id: 1, + start_date: ~D[2026-02-24], + start_time: "11:00:00", + revenue: :REVENUE, + stop_sequence: 1, + arrived: 1_771_968_000, + departed: 1_771_968_100 + }, + %StopEvent{ + id: "trip2-route1-v5-2", + vehicle_id: "v5", + trip_id: "trip2", + route_id: "route1", + stop_id: "stop2", + direction_id: 1, + start_date: ~D[2026-02-24], + start_time: "11:00:00", + revenue: :REVENUE, + stop_sequence: 2, + arrived: 1_771_968_100, + departed: 1_771_968_200 + } + ] + + State.StopEvent.new_state(events) + {:ok, %{}} + end + + test "selects most selective filter when multiple filters provided" do + # vehicle_id (1 match) should be chosen over route_id (5 matches) + result = filter_by(%{vehicle_ids: ["v1"], route_ids: ["route1"]}) + assert length(result) == 1 + assert hd(result).vehicle_id == "v1" + end + + test "handles single filter efficiently without MapSet overhead" do + # Single filter should work without creating MapSets + result = filter_by(%{vehicle_ids: ["v1"]}) + assert length(result) == 1 + assert hd(result).vehicle_id == "v1" + end + + test "handles single filter with direction_id" do + # Single indexed filter + direction should work efficiently + result = filter_by(%{vehicle_ids: ["v1"], direction_id: 0}) + assert length(result) == 1 + assert hd(result).vehicle_id == "v1" + assert hd(result).direction_id == 0 + end + + test "handles direction_id only filter" do + # Direction only should still work (full scan) + result = filter_by(%{direction_id: 0}) + assert length(result) == 3 + assert Enum.all?(result, fn e -> e.direction_id == 0 end) + end + end + describe "by_id/1" do test "returns stop event by id" do stop_event = %StopEvent{ @@ -227,7 +397,7 @@ defmodule State.StopEventTest do start_time: "10:00:00", revenue: :REVENUE, stop_id: "stop1", - current_stop_sequence: 1, + stop_sequence: 1, arrived: 1_771_966_486, departed: 1_771_967_246 } diff --git a/apps/state_mediator/lib/state_mediator.ex b/apps/state_mediator/lib/state_mediator.ex index 8ff16ecca..89a61492e 100644 --- a/apps/state_mediator/lib/state_mediator.ex +++ b/apps/state_mediator/lib/state_mediator.ex @@ -169,7 +169,7 @@ defmodule StateMediator do spec_id: :stop_event_mediator, bucket_arn: app_value(:stop_events, :s3_bucket), object: app_value(:stop_events, :s3_object), - interval: 60 * 1_000, + interval: 3 * 1_000, sync_timeout: 30_000, state: State.StopEvent ] From a38b4a215d51dff26bd39e67c15f4115c6b89539 Mon Sep 17 00:00:00 2001 From: "Corey N. Runkel" Date: Thu, 5 Mar 2026 13:19:05 -0500 Subject: [PATCH 4/6] Conform model typespec to actual behavior --- apps/model/lib/model/stop_event.ex | 10 ++++---- apps/parse/lib/parse/stop_events.ex | 10 ++++---- apps/parse/test/parse/stop_events_test.exs | 28 ++++++++++++++-------- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/apps/model/lib/model/stop_event.ex b/apps/model/lib/model/stop_event.ex index ec7177822..2cf3d47fd 100644 --- a/apps/model/lib/model/stop_event.ex +++ b/apps/model/lib/model/stop_event.ex @@ -38,9 +38,11 @@ defmodule Model.StopEvent do [GTFS Realtime `FeedMesage` `FeedEntity` `TripUpdate` `StopTimeUpdate` `stop_id`](https://github.com/google/transit/blob/master/gtfs-realtime/spec/en/reference.md#message-stoptimeupdate). * `:stop_sequence` - The sequence of the stop along the `trip_id`. The stop sequence increases monotonically but values need not be consecutive. See [GTFS `stop_times.txt` `stop_sequence`](https://github.com/google/transit/blob/master/gtfs/spec/en/reference.md#stop_timestxt). - * `:arrived` - When the vehicle arrived at the stop as seconds since Unix epoch in timezone `America/New_York`. `nil` if the first stop (`stop_id`) on the `trip_id`. - * `:departed` - When the vehicle arrived at the stop as seconds since Unix epoch in timezone `America/New_York`. `nil` if the last stop (`stop_id`) on the `trip_id` + * `:arrived` - When the vehicle arrived at the stop as seconds since Unix epoch (UTC). `nil` if the first stop (`stop_id`) on the `trip_id`. + * `:departed` - When the vehicle departed from the stop as seconds since Unix epoch (UTC). `nil` if the last stop (`stop_id`) on the `trip_id`. """ + @type unix_timestamp :: non_neg_integer + @type t :: %__MODULE__{ id: String.t(), vehicle_id: Model.Vehicle.id(), @@ -52,8 +54,8 @@ defmodule Model.StopEvent do stop_id: Model.Stop.id(), stop_sequence: non_neg_integer, revenue: :REVENUE | :NON_REVENUE, - arrived: DateTime.t() | nil, - departed: DateTime.t() | nil + arrived: unix_timestamp | nil, + departed: unix_timestamp | nil } @spec trip_id(t) :: Model.Trip.id() diff --git a/apps/parse/lib/parse/stop_events.ex b/apps/parse/lib/parse/stop_events.ex index edf1a927c..1c7d92aec 100644 --- a/apps/parse/lib/parse/stop_events.ex +++ b/apps/parse/lib/parse/stop_events.ex @@ -67,10 +67,8 @@ defmodule Parse.StopEvents do defp parse_stop_event( %{ "stop_id" => stop_id, - "stop_sequence" => stop_sequence, - "arrived" => arrived, - "departed" => departed - }, + "stop_sequence" => stop_sequence + } = stop_event, trip_data ) do [ @@ -85,8 +83,8 @@ defmodule Parse.StopEvents do revenue: trip_data.revenue, stop_id: stop_id, stop_sequence: stop_sequence, - arrived: arrived, - departed: departed + arrived: Map.get(stop_event, "arrived"), + departed: Map.get(stop_event, "departed") } ] end diff --git a/apps/parse/test/parse/stop_events_test.exs b/apps/parse/test/parse/stop_events_test.exs index d9ccfa3c7..87aa7f855 100644 --- a/apps/parse/test/parse/stop_events_test.exs +++ b/apps/parse/test/parse/stop_events_test.exs @@ -104,23 +104,31 @@ defmodule Parse.StopEventsTest do assert length(result) == 1 end + test "parses stop events with optional arrived/departed fields" do + ndjson = """ + {"id":"first-stop","timestamp":1771968343,"start_date":"20260224","trip_id":"arrived","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"departed":1771967246}]} + {"id":"last-stop","timestamp":1771968343,"start_date":"20260224","trip_id":"departed","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":1771966486}]} + {"id":"middle-stop","timestamp":1771968343,"start_date":"20260224","trip_id":"both-times","vehicle_id":"v2","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop2","stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + """ + + result = parse(ndjson) + + assert [ + %StopEvent{trip_id: "arrived", arrived: nil, departed: 1_771_967_246}, + %StopEvent{trip_id: "departed", arrived: 1_771_966_486, departed: nil}, + %StopEvent{trip_id: "both-times", arrived: 1_771_966_486, departed: 1_771_967_246} + ] = result + end + test "logs and ignores lines with missing required fields" do ndjson = """ - {"id":"missing-times","timestamp":1771968343,"start_date":"20260224","trip_id":"missing","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1}]} - {"id":"valid-has-arrived","timestamp":1771968343,"start_date":"20260224","trip_id":"arrived","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":1771966486}]} - {"id":"valid-has-departed","timestamp":1771968343,"start_date":"20260224","trip_id":"departed","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"departed":1771967246}]} - {"id":"valid-has-both-times","timestamp":1771968343,"start_date":"20260224","trip_id":"both-times","vehicle_id":"v2","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop2","stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + {"id":"missing-stop-id","timestamp":1771968343,"start_date":"20260224","trip_id":"missing","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} """ log = capture_log(fn -> result = parse(ndjson) - - assert [ - %StopEvent{trip_id: "both-times"}, - %StopEvent{trip_id: "arrived"}, - %StopEvent{trip_id: "departed"} - ] = result + assert result == [] end) assert log =~ "missing_fields" From 46655fada0dc40662fb610125a4853546462bd0c Mon Sep 17 00:00:00 2001 From: "Corey N. Runkel" Date: Thu, 5 Mar 2026 17:44:27 -0500 Subject: [PATCH 5/6] Address credo failures --- .../test/api_web/controllers/stop_event_controller_test.exs | 2 +- apps/state/lib/state/stop_event.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs b/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs index 90725fa0a..fc2dd4a47 100644 --- a/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs +++ b/apps/api_web/test/api_web/controllers/stop_event_controller_test.exs @@ -554,7 +554,7 @@ defmodule ApiWeb.StopEventControllerTest do ) response = json_response(conn, 200)["data"] - ids = Enum.map(response, & &1["id"]) |> Enum.sort() + ids = response |> Enum.map(& &1["id"]) |> Enum.sort() # Both trip1-route1-stop1 and trip2-route2-stop1 match the filters assert ids == ["trip1-route1-v1-1", "trip2-route2-v2-1"] end diff --git a/apps/state/lib/state/stop_event.ex b/apps/state/lib/state/stop_event.ex index 8f7ead7ea..1b3b2475b 100644 --- a/apps/state/lib/state/stop_event.ex +++ b/apps/state/lib/state/stop_event.ex @@ -8,8 +8,8 @@ defmodule State.StopEvent do recordable: Model.StopEvent alias Model.Route - alias Model.StopEvent alias Model.Stop + alias Model.StopEvent alias Model.Trip alias Model.Vehicle From f79b4653c0c6551d13beaf835b0414c42fbc4ad4 Mon Sep 17 00:00:00 2001 From: "Corey N. Runkel" Date: Fri, 6 Mar 2026 10:33:17 -0500 Subject: [PATCH 6/6] Expect unnested record structure --- apps/parse/lib/parse/stop_events.ex | 109 ++++++++------------- apps/parse/test/parse/stop_events_test.exs | 32 +++--- 2 files changed, 55 insertions(+), 86 deletions(-) diff --git a/apps/parse/lib/parse/stop_events.ex b/apps/parse/lib/parse/stop_events.ex index 1c7d92aec..896e90f0f 100644 --- a/apps/parse/lib/parse/stop_events.ex +++ b/apps/parse/lib/parse/stop_events.ex @@ -11,7 +11,8 @@ defmodule Parse.StopEvents do def parse(body) do body |> String.split("\n", trim: true) - |> Enum.flat_map(&parse_line/1) + |> Enum.map(&parse_line/1) + |> Enum.reject(&is_nil/1) end defp parse_line(line) do @@ -21,81 +22,57 @@ defmodule Parse.StopEvents do e -> Logger.warning("#{__MODULE__} decode_error e=#{inspect(e)}") - [] + nil end end - defp parse_record(%{ - "start_date" => start_date, - "trip_id" => trip_id, - "vehicle_id" => vehicle_id, - "direction_id" => direction_id, - "route_id" => route_id, - "start_time" => start_time, - "revenue" => revenue, - "stop_events" => stop_events - }) - when is_list(stop_events) do - with {:ok, date} <- parse_date(start_date), - {:ok, revenue_atom} <- parse_revenue(revenue) do - Enum.flat_map(stop_events, fn stop_event -> - parse_stop_event(stop_event, %{ - start_date: date, - trip_id: trip_id, - vehicle_id: vehicle_id, - direction_id: direction_id, - route_id: route_id, - start_time: start_time, - revenue: revenue_atom - }) - end) - else - error -> - Logger.warning( - "#{__MODULE__} parse_error error=#{inspect(error)} trip_id=#{trip_id} vehicle_id=#{vehicle_id}" - ) - - [] - end - end - - defp parse_record(record) do - Logger.warning("#{__MODULE__} parse_error error=missing_fields #{inspect(record)}") - [] - end - - defp parse_stop_event( + defp parse_record( %{ + "start_date" => start_date, + "id" => id, + "trip_id" => trip_id, + "vehicle_id" => vehicle_id, + "direction_id" => direction_id, + "route_id" => route_id, + "start_time" => start_time, + "revenue" => revenue, "stop_id" => stop_id, "stop_sequence" => stop_sequence - } = stop_event, - trip_data + } = record ) do - [ - %Model.StopEvent{ - id: build_composite_key(trip_data, stop_sequence), - vehicle_id: trip_data.vehicle_id, - start_date: trip_data.start_date, - trip_id: trip_data.trip_id, - direction_id: trip_data.direction_id, - route_id: trip_data.route_id, - start_time: trip_data.start_time, - revenue: trip_data.revenue, - stop_id: stop_id, - stop_sequence: stop_sequence, - arrived: Map.get(stop_event, "arrived"), - departed: Map.get(stop_event, "departed") - } - ] - end + case parse_date(start_date) do + {:ok, date} -> + case parse_revenue(revenue) do + {:ok, revenue_atom} -> + %Model.StopEvent{ + id: id, + vehicle_id: vehicle_id, + start_date: date, + trip_id: trip_id, + direction_id: direction_id, + route_id: route_id, + start_time: start_time, + revenue: revenue_atom, + stop_id: stop_id, + stop_sequence: stop_sequence, + arrived: Map.get(record, "arrived"), + departed: Map.get(record, "departed") + } + + {:error, reason} -> + Logger.warning("#{__MODULE__} parse_error error=#{reason} record=#{inspect(record)}") + nil + end - defp parse_stop_event(stop_event, _trip_data) do - Logger.warning("#{__MODULE__} parse_error error=missing_fields #{inspect(stop_event)}") - [] + {:error, reason} -> + Logger.warning("#{__MODULE__} parse_error error=#{reason} record=#{inspect(record)}") + nil + end end - defp build_composite_key(trip_data, stop_sequence) do - "#{trip_data.trip_id}-#{trip_data.route_id}-#{trip_data.vehicle_id}-#{stop_sequence}" + defp parse_record(record) do + Logger.warning("#{__MODULE__} parse_error error=missing_fields #{inspect(record)}") + nil end defp parse_date(<>) do diff --git a/apps/parse/test/parse/stop_events_test.exs b/apps/parse/test/parse/stop_events_test.exs index 87aa7f855..e5f23b4c2 100644 --- a/apps/parse/test/parse/stop_events_test.exs +++ b/apps/parse/test/parse/stop_events_test.exs @@ -8,8 +8,9 @@ defmodule Parse.StopEventsTest do describe "parse" do test "parses valid NDJSON data with multiple stop events" do ndjson = """ - {"id":"73885810-64-y2071","timestamp":1771968343,"start_date":"20260224","trip_id":"73885810","vehicle_id":"y2071","direction_id":0,"route_id":"64","start_time":"16:07:00","revenue":true,"stop_events":[{"stop_id":"2231","stop_sequence":1,"arrived":1771966486,"departed":1771967246},{"stop_id":"12232","stop_sequence":2,"arrived":1771967286,"departed":1771967333}]} - {"id":"73221192-Green-E-G-10077","timestamp":1771950045,"start_date":"20260224","trip_id":"73221192","vehicle_id":"G-10077","direction_id":0,"route_id":"Green-E","start_time":"10:16:00","revenue":true,"stop_events":[{"stop_id":"70512","stop_sequence":4,"arrived":1771946303,"departed":1771946479}]} + {"id":"73885810-64-y2071-1","timestamp":1771968343,"start_date":"20260224","trip_id":"73885810","vehicle_id":"y2071","direction_id":0,"route_id":"64","start_time":"16:07:00","revenue":true,"stop_id":"2231","stop_sequence":1,"arrived":1771966486,"departed":1771967246} + {"id":"73885810-64-y2071-2","timestamp":1771968343,"start_date":"20260224","trip_id":"73885810","vehicle_id":"y2071","direction_id":0,"route_id":"64","start_time":"16:07:00","revenue":true,"stop_id":"12232","stop_sequence":2,"arrived":1771967286,"departed":1771967333} + {"id":"73221192-Green-E-G-10077-4","timestamp":1771950045,"start_date":"20260224","trip_id":"73221192","vehicle_id":"G-10077","direction_id":0,"route_id":"Green-E","start_time":"10:16:00","revenue":true,"stop_id":"70512","stop_sequence":4,"arrived":1771946303,"departed":1771946479} """ result = parse(ndjson) @@ -64,7 +65,7 @@ defmodule Parse.StopEventsTest do test "handles null departed times for last stop" do ndjson = """ - {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":1771966486,"departed":null}]} + {"id":"test-trip-1","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_id":"stop1","stop_sequence":1,"arrived":1771966486,"departed":null} """ result = parse(ndjson) @@ -74,7 +75,7 @@ defmodule Parse.StopEventsTest do test "handles null arrived times for first stop" do ndjson = """ - {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":null,"departed":1771967246}]} + {"id":"test-trip-1","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_id":"stop1","stop_sequence":1,"arrived":null,"departed":1771967246} """ result = parse(ndjson) @@ -84,7 +85,7 @@ defmodule Parse.StopEventsTest do test "handles non-revenue trips" do ndjson = """ - {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":false,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + {"id":"test-trip-1","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":false,"stop_id":"stop1","stop_sequence":1,"arrived":1771966486,"departed":1771967246} """ result = parse(ndjson) @@ -95,7 +96,7 @@ defmodule Parse.StopEventsTest do test "ignores empty lines in NDJSON" do ndjson = """ - {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + {"id":"test-trip-1","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_id":"stop1","stop_sequence":1,"arrived":1771966486,"departed":1771967246} """ @@ -106,9 +107,9 @@ defmodule Parse.StopEventsTest do test "parses stop events with optional arrived/departed fields" do ndjson = """ - {"id":"first-stop","timestamp":1771968343,"start_date":"20260224","trip_id":"arrived","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"departed":1771967246}]} - {"id":"last-stop","timestamp":1771968343,"start_date":"20260224","trip_id":"departed","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":1771966486}]} - {"id":"middle-stop","timestamp":1771968343,"start_date":"20260224","trip_id":"both-times","vehicle_id":"v2","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop2","stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + {"id":"first-stop-1","timestamp":1771968343,"start_date":"20260224","trip_id":"arrived","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_id":"stop1","stop_sequence":1,"departed":1771967246} + {"id":"last-stop-1","timestamp":1771968343,"start_date":"20260224","trip_id":"departed","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_id":"stop1","stop_sequence":1,"arrived":1771966486} + {"id":"middle-stop-1","timestamp":1771968343,"start_date":"20260224","trip_id":"both-times","vehicle_id":"v2","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_id":"stop2","stop_sequence":1,"arrived":1771966486,"departed":1771967246} """ result = parse(ndjson) @@ -122,7 +123,7 @@ defmodule Parse.StopEventsTest do test "logs and ignores lines with missing required fields" do ndjson = """ - {"id":"missing-stop-id","timestamp":1771968343,"start_date":"20260224","trip_id":"missing","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + {"id":"missing-stop-id-1","timestamp":1771968343,"start_date":"20260224","trip_id":"missing","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_sequence":1,"arrived":1771966486,"departed":1771967246} """ log = @@ -136,7 +137,7 @@ defmodule Parse.StopEventsTest do test "logs and ignores lines with invalid date format" do ndjson = """ - {"id":"test-trip","timestamp":1771968343,"start_date":"invalid","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[{"stop_id":"stop1","stop_sequence":1,"arrived":1771966486,"departed":1771967246}]} + {"id":"test-trip-1","timestamp":1771968343,"start_date":"invalid","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_id":"stop1","stop_sequence":1,"arrived":1771966486,"departed":1771967246} """ log = @@ -155,14 +156,5 @@ defmodule Parse.StopEventsTest do assert log =~ "decode_error" end - - test "handles trip with empty stop_events array" do - ndjson = """ - {"id":"test-trip","timestamp":1771968343,"start_date":"20260224","trip_id":"test","vehicle_id":"v1","direction_id":0,"route_id":"1","start_time":"10:00:00","revenue":true,"stop_events":[]} - """ - - result = parse(ndjson) - assert result == [] - end end end