From 2d358776a6c278e027fb701a03ba45769969910a Mon Sep 17 00:00:00 2001 From: Lorenzo Delgado Date: Sat, 28 Mar 2026 10:49:17 +0100 Subject: [PATCH] refactor(controller): merge admin-api crate into controller Eliminate the admin-api indirection layer since controller is its sole consumer, absorbing routing, context, and handler modules directly. - Move `ctx`, `build_info`, `handlers` modules from `admin-api` into `controller` - Extract `router()` and OpenAPI spec generation into new `router.rs` module - Merge all `admin-api` dependencies and `utoipa` feature into controller - Move gen crate to `controller/gen/` and rename to `amp-controller-admin-api-gen` - Update justfile openapi gen tasks to reference new crate name Signed-off-by: Lorenzo Delgado --- Cargo.lock | 44 +++++--------- Cargo.toml | 3 +- crates/core/monitoring/src/logging.rs | 1 - crates/services/admin-api/Cargo.toml | 25 -------- crates/services/admin-api/README.md | 59 ------------------- crates/services/controller/Cargo.toml | 10 +++- .../SPEC_DESCRIPTION.md | 0 .../{admin-api => controller}/gen/Cargo.toml | 4 +- .../{admin-api => controller}/gen/README.md | 0 .../{admin-api => controller}/gen/build.rs | 2 +- .../{admin-api => controller}/gen/src/lib.rs | 0 .../src/build_info.rs | 0 .../{admin-api => controller}/src/ctx.rs | 0 .../{admin-api => controller}/src/handlers.rs | 0 .../src/handlers/error.rs | 0 crates/services/controller/src/lib.rs | 9 ++- .../src/lib.rs => controller/src/router.rs} | 14 ++--- crates/services/controller/src/service.rs | 5 +- justfile | 6 +- 19 files changed, 43 insertions(+), 139 deletions(-) delete mode 100644 crates/services/admin-api/Cargo.toml delete mode 100644 crates/services/admin-api/README.md rename crates/services/{admin-api => controller}/SPEC_DESCRIPTION.md (100%) rename crates/services/{admin-api => controller}/gen/Cargo.toml (85%) rename crates/services/{admin-api => controller}/gen/README.md (100%) rename crates/services/{admin-api => controller}/gen/build.rs (93%) rename crates/services/{admin-api => controller}/gen/src/lib.rs (100%) rename crates/services/{admin-api => controller}/src/build_info.rs (100%) rename crates/services/{admin-api => controller}/src/ctx.rs (100%) rename crates/services/{admin-api => controller}/src/handlers.rs (100%) rename crates/services/{admin-api => controller}/src/handlers/error.rs (100%) rename crates/services/{admin-api/src/lib.rs => controller/src/router.rs} (97%) diff --git a/Cargo.lock b/Cargo.lock index 0b8584e15..53db1b0f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,35 +24,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "admin-api" -version = "0.1.0" -dependencies = [ - "amp-controller-admin-datasets", - "amp-controller-admin-jobs", - "amp-controller-admin-providers", - "amp-controller-admin-system", - "amp-controller-admin-tables", - "amp-data-store", - "amp-datasets-registry", - "amp-providers-registry", - "amp-worker-core", - "async-trait", - "axum", - "common", - "metadata-db", - "serde", - "utoipa", -] - -[[package]] -name = "admin-api-gen" -version = "0.1.0" -dependencies = [ - "admin-api", - "serde_json", -] - [[package]] name = "admin-client" version = "0.1.0" @@ -966,6 +937,14 @@ dependencies = [ "serde_json", ] +[[package]] +name = "amp-controller-admin-api-gen" +version = "0.1.0" +dependencies = [ + "controller", + "serde_json", +] + [[package]] name = "amp-controller-admin-datasets" version = "0.1.0" @@ -3475,8 +3454,11 @@ checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" name = "controller" version = "0.1.0" dependencies = [ - "admin-api", + "amp-controller-admin-datasets", "amp-controller-admin-jobs", + "amp-controller-admin-providers", + "amp-controller-admin-system", + "amp-controller-admin-tables", "amp-data-store", "amp-datasets-registry", "amp-providers-registry", @@ -3489,10 +3471,12 @@ dependencies = [ "monitoring", "opentelemetry-instrumentation-tower", "rand 0.9.2", + "serde", "thiserror 2.0.18", "tokio", "tower-http", "tracing", + "utoipa", "worker", ] diff --git a/Cargo.toml b/Cargo.toml index a5532cf21..583a07714 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,9 +56,8 @@ members = [ "crates/controller-admin-providers", "crates/controller-admin-system", "crates/controller-admin-tables", - "crates/services/admin-api", - "crates/services/admin-api/gen", "crates/services/controller", + "crates/services/controller/gen", "crates/services/server", "crates/services/metadata-db-postgres", "crates/services/worker", diff --git a/crates/core/monitoring/src/logging.rs b/crates/core/monitoring/src/logging.rs index aeaaec789..14f6be44a 100644 --- a/crates/core/monitoring/src/logging.rs +++ b/crates/core/monitoring/src/logging.rs @@ -63,7 +63,6 @@ pub fn init_with_telemetry(url: &str, trace_ratio: f64) -> telemetry::traces::Re /// List of crates in the workspace. const AMP_CRATES: &[&str] = &[ - "admin_api", "admin_client", "amp_client", "amp_config", diff --git a/crates/services/admin-api/Cargo.toml b/crates/services/admin-api/Cargo.toml deleted file mode 100644 index f00acdc02..000000000 --- a/crates/services/admin-api/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "admin-api" -version.workspace = true -license-file.workspace = true -edition.workspace = true - -[features] -utoipa = ["dep:utoipa", "amp-controller-admin-datasets/utoipa", "amp-controller-admin-jobs/utoipa", "amp-controller-admin-providers/utoipa", "amp-controller-admin-system/utoipa", "amp-controller-admin-tables/utoipa"] - -[dependencies] -amp-controller-admin-datasets = { path = "../../controller-admin-datasets" } -amp-controller-admin-jobs = { path = "../../controller-admin-jobs" } -amp-controller-admin-providers = { path = "../../controller-admin-providers" } -amp-controller-admin-system = { path = "../../controller-admin-system" } -amp-controller-admin-tables = { path = "../../controller-admin-tables" } -amp-data-store = { path = "../../core/data-store" } -amp-datasets-registry = { path = "../../core/datasets-registry" } -amp-providers-registry = { path = "../../core/providers-registry" } -amp-worker-core = { path = "../../core/worker-core" } -async-trait.workspace = true -axum.workspace = true -common = { path = "../../core/common" } -metadata-db = { path = "../../core/metadata-db" } -serde.workspace = true -utoipa = { version = "5.4.0", features = ["axum_extras"], optional = true } diff --git a/crates/services/admin-api/README.md b/crates/services/admin-api/README.md deleted file mode 100644 index 61a1981ee..000000000 --- a/crates/services/admin-api/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Admin API - -## Summary - -The Admin API provides a RESTful HTTP interface for managing Amp's ETL operations. This crate implements an Axum-based web server that exposes endpoints for dataset management, job control, storage location administration, file management, and provider configuration. - -The API serves as the primary administrative interface for monitoring and controlling the Amp data pipeline, allowing operators to: -- Deploy and manage datasets with versioning support -- Trigger and monitor data extraction jobs -- Manage distributed storage locations -- Configure external data providers (EVM RPC, Firehose) -- Perform operations on Parquet files and their metadata - -## API Documentation - -Complete API documentation is available in the generated OpenAPI specification at [`docs/schemas/openapi/admin.spec.json`](../../docs/schemas/openapi/admin.spec.json). - -The OpenAPI spec provides comprehensive documentation for all endpoints, including: -- Request/response schemas -- Pagination details -- Error handling -- Usage examples -- Data format specifications - -## Crate Structure - -The admin-api crate follows a modular handler-based architecture: - -``` -src/ -├── ctx.rs # Application context with shared state -├── handlers.rs # Handler module declarations -├── handlers/ -│ ├── common.rs # Shared utilities for handlers -│ ├── error.rs # Error response types -│ ├── datasets/ # Dataset management endpoints -│ ├── files/ # File management endpoints -│ ├── jobs/ # Job management endpoints -│ ├── locations/ # Storage location endpoints -│ └── providers/ # Provider configuration endpoints -├── lib.rs # Main server setup and routing -└── scheduler.rs # Job scheduling utilities -``` - -### Handler Registration - -Handlers are registered in [`src/lib.rs`](src/lib.rs) using Axum's routing system. Each endpoint is mapped to its corresponding handler function using HTTP method routing helpers (`get()`, `post()`, `put()`, `delete()`). - -## OpenAPI Spec Generation - -OpenAPI specifications for the Admin API can be generated from the code annotations. This generates comprehensive API documentation including all endpoints, schemas, and usage examples. - -To generate the OpenAPI specification, run: - -```bash -just gen-admin-api-openapi-spec -``` - -This will generate the OpenAPI spec from the handler utoipa annotations and copy it to `docs/schemas/openapi/admin.spec.json`. diff --git a/crates/services/controller/Cargo.toml b/crates/services/controller/Cargo.toml index d7748845a..7db1cc764 100644 --- a/crates/services/controller/Cargo.toml +++ b/crates/services/controller/Cargo.toml @@ -4,9 +4,15 @@ edition = "2024" version.workspace = true license-file.workspace = true +[features] +utoipa = ["dep:utoipa", "amp-controller-admin-datasets/utoipa", "amp-controller-admin-jobs/utoipa", "amp-controller-admin-providers/utoipa", "amp-controller-admin-system/utoipa", "amp-controller-admin-tables/utoipa"] + [dependencies] -admin-api = { path = "../admin-api" } +amp-controller-admin-datasets = { path = "../../controller-admin-datasets" } amp-controller-admin-jobs = { path = "../../controller-admin-jobs" } +amp-controller-admin-providers = { path = "../../controller-admin-providers" } +amp-controller-admin-system = { path = "../../controller-admin-system" } +amp-controller-admin-tables = { path = "../../controller-admin-tables" } amp-data-store = { path = "../../core/data-store" } amp-datasets-registry = { version = "0.1.0", path = "../../core/datasets-registry" } amp-providers-registry = { version = "0.1.0", path = "../../core/providers-registry" } @@ -19,8 +25,10 @@ metadata-db = { version = "0.1.0", path = "../../core/metadata-db" } monitoring = { path = "../../core/monitoring" } opentelemetry-instrumentation-tower = { version = "0.17.0", features = ["axum"] } rand.workspace = true +serde.workspace = true thiserror.workspace = true tokio.workspace = true tower-http = { workspace = true, features = ["cors"] } tracing.workspace = true +utoipa = { version = "5.4.0", features = ["axum_extras"], optional = true } worker = { version = "0.1.0", path = "../worker" } diff --git a/crates/services/admin-api/SPEC_DESCRIPTION.md b/crates/services/controller/SPEC_DESCRIPTION.md similarity index 100% rename from crates/services/admin-api/SPEC_DESCRIPTION.md rename to crates/services/controller/SPEC_DESCRIPTION.md diff --git a/crates/services/admin-api/gen/Cargo.toml b/crates/services/controller/gen/Cargo.toml similarity index 85% rename from crates/services/admin-api/gen/Cargo.toml rename to crates/services/controller/gen/Cargo.toml index b16a924b8..889d87888 100644 --- a/crates/services/admin-api/gen/Cargo.toml +++ b/crates/services/controller/gen/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "admin-api-gen" +name = "amp-controller-admin-api-gen" edition.workspace = true version.workspace = true license-file.workspace = true @@ -8,7 +8,7 @@ publish = false # Schema generation dependencies (see README.md "OpenAPI Schema Generation" section) # These dependencies are only included when the gen_openapi_spec cfg flag is enabled [target.'cfg(gen_openapi_spec)'.build-dependencies] -admin-api = { path = "..", features = ["utoipa"] } +controller = { path = "..", features = ["utoipa"] } serde_json.workspace = true [lints.rust] diff --git a/crates/services/admin-api/gen/README.md b/crates/services/controller/gen/README.md similarity index 100% rename from crates/services/admin-api/gen/README.md rename to crates/services/controller/gen/README.md diff --git a/crates/services/admin-api/gen/build.rs b/crates/services/controller/gen/build.rs similarity index 93% rename from crates/services/admin-api/gen/build.rs rename to crates/services/controller/gen/build.rs index 882deb7c1..ce85a654b 100644 --- a/crates/services/admin-api/gen/build.rs +++ b/crates/services/controller/gen/build.rs @@ -6,7 +6,7 @@ fn main() -> Result<(), Box> { ); let out_dir = std::env::var("OUT_DIR")?; - let spec = admin_api::generate_openapi_spec(); + let spec = controller::generate_openapi_spec(); let spec_json = serde_json::to_string_pretty(&spec)?; let spec_path = format!("{out_dir}/openapi.spec.json"); std::fs::write(&spec_path, spec_json)?; diff --git a/crates/services/admin-api/gen/src/lib.rs b/crates/services/controller/gen/src/lib.rs similarity index 100% rename from crates/services/admin-api/gen/src/lib.rs rename to crates/services/controller/gen/src/lib.rs diff --git a/crates/services/admin-api/src/build_info.rs b/crates/services/controller/src/build_info.rs similarity index 100% rename from crates/services/admin-api/src/build_info.rs rename to crates/services/controller/src/build_info.rs diff --git a/crates/services/admin-api/src/ctx.rs b/crates/services/controller/src/ctx.rs similarity index 100% rename from crates/services/admin-api/src/ctx.rs rename to crates/services/controller/src/ctx.rs diff --git a/crates/services/admin-api/src/handlers.rs b/crates/services/controller/src/handlers.rs similarity index 100% rename from crates/services/admin-api/src/handlers.rs rename to crates/services/controller/src/handlers.rs diff --git a/crates/services/admin-api/src/handlers/error.rs b/crates/services/controller/src/handlers/error.rs similarity index 100% rename from crates/services/admin-api/src/handlers/error.rs rename to crates/services/controller/src/handlers/error.rs diff --git a/crates/services/controller/src/lib.rs b/crates/services/controller/src/lib.rs index a06cb0a94..1ab444f07 100644 --- a/crates/services/controller/src/lib.rs +++ b/crates/services/controller/src/lib.rs @@ -2,8 +2,13 @@ //! //! The controller service provides the admin API for managing Amp operations. +pub mod build_info; +pub mod ctx; +pub mod handlers; +mod router; mod scheduler; pub mod service; -// Re-export build_info to avoid code duplication -pub use admin_api::build_info; +#[cfg(feature = "utoipa")] +pub use router::generate_openapi_spec; +pub(crate) use router::router; diff --git a/crates/services/admin-api/src/lib.rs b/crates/services/controller/src/router.rs similarity index 97% rename from crates/services/admin-api/src/lib.rs rename to crates/services/controller/src/router.rs index 8d89112d8..04509b508 100644 --- a/crates/services/admin-api/src/lib.rs +++ b/crates/services/controller/src/router.rs @@ -1,21 +1,15 @@ -//! Amp Admin API +//! Amp Admin API router and OpenAPI specification use std::sync::Arc; use axum::Router; -pub mod build_info; -pub mod ctx; -pub mod handlers; - -use ctx::Ctx; - -use crate::ctx::{RevisionGuardImpl, WorkerServiceImpl}; +use crate::ctx::{Ctx, RevisionGuardImpl, WorkerServiceImpl}; /// Create the admin API router with all routes registered /// /// Returns a router configured with all admin API endpoints. -pub fn router(ctx: Ctx) -> Router<()> { +pub(crate) fn router(ctx: Ctx) -> Router<()> { let tables_ctx = amp_controller_admin_tables::ctx::Ctx { metadata_db: ctx.metadata_db.clone(), datasets_registry: ctx.datasets_registry.clone(), @@ -114,7 +108,7 @@ pub fn router(ctx: Ctx) -> Router<()> { ), components(schemas( // Common schemas - handlers::error::ErrorResponse, + crate::handlers::error::ErrorResponse, // Manifest schemas amp_controller_admin_datasets::manifests::handlers::list_all::ManifestsResponse, amp_controller_admin_datasets::manifests::handlers::list_all::ManifestInfo, diff --git a/crates/services/controller/src/service.rs b/crates/services/controller/src/service.rs index 276d7e4c7..a6db9577e 100644 --- a/crates/services/controller/src/service.rs +++ b/crates/services/controller/src/service.rs @@ -1,6 +1,5 @@ use std::{future::Future, net::SocketAddr, sync::Arc, time::Duration}; -use admin_api::ctx::Ctx; use amp_data_store::DataStore; use amp_datasets_registry::DatasetsRegistry; use amp_providers_registry::ProvidersRegistry; @@ -17,7 +16,7 @@ use opentelemetry_instrumentation_tower::HTTPMetricsLayerBuilder; use tokio::{net::TcpListener, time::MissedTickBehavior}; use tower_http::cors::CorsLayer; -use crate::{build_info::BuildInfo, scheduler::Scheduler}; +use crate::{build_info::BuildInfo, ctx::Ctx, scheduler::Scheduler}; /// Reconciliation interval for failed job retries /// @@ -61,7 +60,7 @@ pub async fn new( // Create controller router with health check endpoint let mut app = Router::new() .route("/healthz", get(|| async { StatusCode::OK })) - .merge(admin_api::router(ctx)); + .merge(crate::router(ctx)); // Add OpenTelemetry HTTP metrics middleware if meter is provided if let Some(meter) = meter { diff --git a/justfile b/justfile index 1f88e13dc..0b86c500a 100644 --- a/justfile +++ b/justfile @@ -261,7 +261,7 @@ gen: echo " {{GEN_PROVIDER_SCHEMAS_OUTDIR}}/static.spec.json" # Admin API (OpenAPI spec) - cp -f $(ls -t target/debug/build/admin-api-gen-*/out/openapi.spec.json | head -1) {{GEN_OPENAPI_SCHEMAS_OUTDIR}}/admin.spec.json + cp -f $(ls -t target/debug/build/amp-controller-admin-api-gen-*/out/openapi.spec.json | head -1) {{GEN_OPENAPI_SCHEMAS_OUTDIR}}/admin.spec.json echo " {{GEN_OPENAPI_SCHEMAS_OUTDIR}}/admin.spec.json" ### ampd Config @@ -410,9 +410,9 @@ gen-static-provider-schema DEST_DIR=GEN_PROVIDER_SCHEMAS_OUTDIR: # Generate the admin API OpenAPI specification [group: 'codegen'] gen-admin-api-openapi-spec DEST_DIR=GEN_OPENAPI_SCHEMAS_OUTDIR: - RUSTFLAGS="--cfg gen_openapi_spec" cargo check -p admin-api-gen + RUSTFLAGS="--cfg gen_openapi_spec" cargo check -p amp-controller-admin-api-gen @mkdir -p {{DEST_DIR}} - @cp -f $(ls -t target/debug/build/admin-api-gen-*/out/openapi.spec.json | head -1) {{DEST_DIR}}/admin.spec.json + @cp -f $(ls -t target/debug/build/amp-controller-admin-api-gen-*/out/openapi.spec.json | head -1) {{DEST_DIR}}/admin.spec.json @echo "Schema generated and copied to {{DEST_DIR}}/admin.spec.json" ### Worker