From 8c1331bddcda50d8439e09d78ecec0f873de417b Mon Sep 17 00:00:00 2001 From: Lorenzo Delgado Date: Sat, 28 Mar 2026 01:37:34 +0100 Subject: [PATCH] refactor(admin-api): extract datasets and manifests into controller crate Continue admin-api decomposition by moving dataset and manifest handlers into a dedicated controller-admin-datasets crate with its own context and error types. - Create `controller-admin-datasets` crate with dataset, manifest, and schema handlers - Move `restore` and `restore_table` handlers to `controller-admin-tables` (table-scoped operations) - Introduce `Ctx` and `ErrorResponse`/`IntoErrorResponse` types local to the new crate - Wire new crate router into admin-api via `merge` with dedicated state - Update utoipa OpenAPI paths and schema references to point to extracted modules Signed-off-by: Lorenzo Delgado --- Cargo.lock | 28 +++- Cargo.toml | 1 + crates/controller-admin-datasets/Cargo.toml | 27 ++++ .../src}/common.rs | 122 +----------------- crates/controller-admin-datasets/src/ctx.rs | 18 +++ .../controller-admin-datasets/src/datasets.rs | 1 + .../src/datasets/handlers.rs | 8 ++ .../src/datasets/handlers}/delete.rs | 2 +- .../src/datasets/handlers}/delete_version.rs | 2 +- .../src/datasets/handlers}/get.rs | 2 +- .../src/datasets/handlers}/get_all.rs | 4 +- .../src/datasets/handlers}/get_manifest.rs | 2 +- .../src/datasets/handlers}/list_all.rs | 2 +- .../src/datasets/handlers}/list_versions.rs | 2 +- .../src/datasets/handlers}/register.rs | 18 ++- crates/controller-admin-datasets/src/error.rs | 95 ++++++++++++++ crates/controller-admin-datasets/src/lib.rs | 55 ++++++++ .../src/manifests.rs | 1 + .../src/manifests/handlers.rs} | 0 .../src/manifests/handlers}/delete_by_id.rs | 8 +- .../src/manifests/handlers}/get_by_id.rs | 8 +- .../src/manifests/handlers}/list_all.rs | 4 +- .../src/manifests/handlers}/list_datasets.rs | 2 +- .../src/manifests/handlers}/prune.rs | 4 +- .../src/manifests/handlers}/register.rs | 18 ++- .../src}/schema.rs | 12 +- .../controller-admin-tables/src/datasets.rs | 1 + .../src/datasets/handlers.rs | 2 + .../src/datasets/handlers}/restore.rs | 6 +- .../src/datasets/handlers}/restore_table.rs | 6 +- crates/controller-admin-tables/src/lib.rs | 23 +++- .../controller-admin-tables/src/revisions.rs | 2 +- crates/core/monitoring/src/logging.rs | 1 + crates/services/admin-api/Cargo.toml | 8 +- crates/services/admin-api/src/handlers.rs | 3 - .../admin-api/src/handlers/datasets.rs | 10 -- crates/services/admin-api/src/lib.rs | 121 ++++++----------- 37 files changed, 343 insertions(+), 286 deletions(-) create mode 100644 crates/controller-admin-datasets/Cargo.toml rename crates/{services/admin-api/src/handlers => controller-admin-datasets/src}/common.rs (86%) create mode 100644 crates/controller-admin-datasets/src/ctx.rs create mode 100644 crates/controller-admin-datasets/src/datasets.rs create mode 100644 crates/controller-admin-datasets/src/datasets/handlers.rs rename crates/{services/admin-api/src/handlers/datasets => controller-admin-datasets/src/datasets/handlers}/delete.rs (99%) rename crates/{services/admin-api/src/handlers/datasets => controller-admin-datasets/src/datasets/handlers}/delete_version.rs (99%) rename crates/{services/admin-api/src/handlers/datasets => controller-admin-datasets/src/datasets/handlers}/get.rs (99%) rename crates/{services/admin-api/src/handlers/datasets => controller-admin-datasets/src/datasets/handlers}/get_all.rs (97%) rename crates/{services/admin-api/src/handlers/datasets => controller-admin-datasets/src/datasets/handlers}/get_manifest.rs (99%) rename crates/{services/admin-api/src/handlers/datasets => controller-admin-datasets/src/datasets/handlers}/list_all.rs (98%) rename crates/{services/admin-api/src/handlers/datasets => controller-admin-datasets/src/datasets/handlers}/list_versions.rs (99%) rename crates/{services/admin-api/src/handlers/datasets => controller-admin-datasets/src/datasets/handlers}/register.rs (98%) create mode 100644 crates/controller-admin-datasets/src/error.rs create mode 100644 crates/controller-admin-datasets/src/lib.rs create mode 100644 crates/controller-admin-datasets/src/manifests.rs rename crates/{services/admin-api/src/handlers/manifests.rs => controller-admin-datasets/src/manifests/handlers.rs} (100%) rename crates/{services/admin-api/src/handlers/manifests => controller-admin-datasets/src/manifests/handlers}/delete_by_id.rs (98%) rename crates/{services/admin-api/src/handlers/manifests => controller-admin-datasets/src/manifests/handlers}/get_by_id.rs (97%) rename crates/{services/admin-api/src/handlers/manifests => controller-admin-datasets/src/manifests/handlers}/list_all.rs (97%) rename crates/{services/admin-api/src/handlers/manifests => controller-admin-datasets/src/manifests/handlers}/list_datasets.rs (99%) rename crates/{services/admin-api/src/handlers/manifests => controller-admin-datasets/src/manifests/handlers}/prune.rs (98%) rename crates/{services/admin-api/src/handlers/manifests => controller-admin-datasets/src/manifests/handlers}/register.rs (95%) rename crates/{services/admin-api/src/handlers => controller-admin-datasets/src}/schema.rs (98%) create mode 100644 crates/controller-admin-tables/src/datasets.rs create mode 100644 crates/controller-admin-tables/src/datasets/handlers.rs rename crates/{services/admin-api/src/handlers/datasets => controller-admin-tables/src/datasets/handlers}/restore.rs (98%) rename crates/{services/admin-api/src/handlers/datasets => controller-admin-tables/src/datasets/handlers}/restore_table.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index 1d396b1b4..766f50659 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,11 +28,11 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" name = "admin-api" version = "0.1.0" dependencies = [ + "amp-controller-admin-datasets", "amp-controller-admin-system", "amp-controller-admin-tables", "amp-data-store", "amp-datasets-registry", - "amp-parquet", "amp-providers-common", "amp-providers-registry", "amp-worker-core", @@ -42,17 +42,13 @@ dependencies = [ "async-trait", "axum", "common", - "datafusion", "datasets-common", "datasets-derived", - "datasets-raw", - "futures", "metadata-db", "monitoring", "serde", "serde_json", "thiserror 2.0.18", - "tokio", "toml", "tracing", "utoipa", @@ -980,6 +976,28 @@ dependencies = [ "serde_json", ] +[[package]] +name = "amp-controller-admin-datasets" +version = "0.1.0" +dependencies = [ + "amp-data-store", + "amp-datasets-registry", + "axum", + "common", + "datafusion", + "datasets-common", + "datasets-derived", + "datasets-raw", + "futures", + "metadata-db", + "monitoring", + "serde", + "serde_json", + "thiserror 2.0.18", + "tracing", + "utoipa", +] + [[package]] name = "amp-controller-admin-system" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index b36aa8309..e9d607517 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ members = [ "crates/extractors/solana-storage-proto", "crates/extractors/tempo", "crates/extractors/tempo/gen", + "crates/controller-admin-datasets", "crates/controller-admin-system", "crates/controller-admin-tables", "crates/services/admin-api", diff --git a/crates/controller-admin-datasets/Cargo.toml b/crates/controller-admin-datasets/Cargo.toml new file mode 100644 index 000000000..62d46f450 --- /dev/null +++ b/crates/controller-admin-datasets/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "amp-controller-admin-datasets" +edition.workspace = true +version.workspace = true +license-file.workspace = true + +[features] +# OpenAPI schema generation via utoipa +utoipa = ["dep:utoipa"] + +[dependencies] +amp-data-store = { path = "../core/data-store" } +amp-datasets-raw = { path = "../core/datasets-raw", package = "datasets-raw" } +amp-datasets-registry = { path = "../core/datasets-registry" } +axum.workspace = true +common = { path = "../core/common" } +datafusion.workspace = true +datasets-common = { path = "../core/datasets-common" } +datasets-derived = { path = "../core/datasets-derived" } +futures.workspace = true +metadata-db = { path = "../core/metadata-db" } +monitoring = { path = "../core/monitoring" } +serde.workspace = true +serde_json = { workspace = true, features = ["raw_value"] } +thiserror.workspace = true +tracing.workspace = true +utoipa = { version = "5.4.0", features = ["axum_extras"], optional = true } diff --git a/crates/services/admin-api/src/handlers/common.rs b/crates/controller-admin-datasets/src/common.rs similarity index 86% rename from crates/services/admin-api/src/handlers/common.rs rename to crates/controller-admin-datasets/src/common.rs index 80093188b..6ee1c6f6a 100644 --- a/crates/services/admin-api/src/handlers/common.rs +++ b/crates/controller-admin-datasets/src/common.rs @@ -1,16 +1,14 @@ -//! Common utilities for HTTP handlers +//! Common utilities for dataset and manifest handlers use std::{ collections::{BTreeMap, BTreeSet}, sync::Arc, }; -use amp_data_store::{DataStore, PhyTableRevision}; use amp_datasets_raw::dataset_kind::{ EvmRpcDatasetKind, FirehoseDatasetKind, SolanaDatasetKind, TempoDatasetKind, }; use amp_datasets_registry::error::ResolveRevisionError; -use amp_parquet::footer::{AmpMetadataFromParquetError, amp_metadata_from_parquet_file}; use common::{ amp_catalog_provider::{AMP_CATALOG_NAME, AmpCatalogProvider, AsyncSchemaProvider}, context::plan::PlanContextBuilder, @@ -34,7 +32,6 @@ use datasets_derived::{ manifest::{TableInput, View}, sorting::{self, CyclicDepError}, }; -use futures::{StreamExt as _, stream}; /// Map of table names to their SQL references (table refs and function refs) using dependency aliases or self-references. type TableReferencesMap = BTreeMap< @@ -688,120 +685,3 @@ pub enum ManifestValidationError { #[error("failed to create session config")] SessionConfig(#[source] datafusion::error::DataFusionError), } - -/// Registers all files in a revision with their Amp-specific metadata. -/// -/// Lists all files in the revision directory in object storage, extracts -/// Parquet metadata including Amp-specific block range information, and -/// registers each file in the metadata database. -/// -/// Files are processed concurrently (up to 16 at a time). -#[tracing::instrument(skip_all, err)] -pub async fn register_revision_files( - store: &DataStore, - revision: &PhyTableRevision, -) -> Result { - let files = store - .list_revision_files_in_object_store(revision) - .await - .map_err(RegisterRevisionFilesError::ListFiles)?; - let total_files = files.len(); - - // Process files in parallel using buffered stream - const CONCURRENT_METADATA_FETCHES: usize = 16; - - let object_store = store.clone(); - let mut file_stream = stream::iter(files.into_iter()) - .map(|object_meta| { - let store = object_store.clone(); - async move { - let (file_name, amp_meta, footer) = - amp_metadata_from_parquet_file(&store, &object_meta) - .await - .map_err(RegisterRevisionFilesError::ReadParquetMetadata)?; - - let parquet_meta_json = serde_json::to_value(amp_meta) - .map_err(RegisterRevisionFilesError::SerializeMetadata)?; - - let object_size = object_meta.size; - let object_e_tag = object_meta.e_tag; - let object_version = object_meta.version; - - Ok(( - file_name, - object_size, - object_e_tag, - object_version, - parquet_meta_json, - footer, - )) - } - }) - .buffered(CONCURRENT_METADATA_FETCHES); - - // Register all files in the metadata database as they complete - while let Some(result) = file_stream.next().await { - let (file_name, object_size, object_e_tag, object_version, parquet_meta_json, footer) = - result?; - store - .register_revision_file( - revision, - &file_name, - object_size, - object_e_tag, - object_version, - parquet_meta_json, - &footer, - ) - .await - .map_err(RegisterRevisionFilesError::RegisterFile)?; - } - - Ok(total_files as i32) -} - -/// Errors that occur when registering revision files -/// -/// This error type is used by [`register_revision_files`]. -#[derive(Debug, thiserror::Error)] -pub enum RegisterRevisionFilesError { - /// Failed to list files in the revision directory - /// - /// This occurs when: - /// - The object storage path for the revision is inaccessible - /// - Network or permission errors when listing objects - /// - The revision directory does not exist in object storage - #[error("Failed to list files in revision")] - ListFiles(#[source] amp_data_store::ListRevisionFilesInObjectStoreError), - - /// Failed to read Amp metadata from parquet file - /// - /// This occurs when extracting Amp-specific metadata from a Parquet file fails. - /// Common causes include: - /// - Corrupted or invalid Parquet file structure - /// - Missing required metadata keys in the file - /// - Incompatible metadata schema version - /// - I/O errors reading from object store - /// - JSON parsing failures in metadata values - /// - /// See `AmpMetadataFromParquetError` for specific error details. - #[error("Failed to read Amp metadata from parquet file")] - ReadParquetMetadata(#[source] AmpMetadataFromParquetError), - - /// Failed to serialize parquet metadata to JSON - /// - /// This occurs when: - /// - The extracted Amp metadata cannot be represented as valid JSON - /// - Serialization encounters unsupported types or values - #[error("Failed to serialize parquet metadata to JSON")] - SerializeMetadata(#[source] serde_json::Error), - - /// Failed to register file in metadata database - /// - /// This occurs when: - /// - Database connection or transaction errors during file registration - /// - Constraint violations when inserting file metadata - /// - Concurrent modification conflicts - #[error("Failed to register file in metadata database")] - RegisterFile(#[source] amp_data_store::RegisterFileError), -} diff --git a/crates/controller-admin-datasets/src/ctx.rs b/crates/controller-admin-datasets/src/ctx.rs new file mode 100644 index 000000000..afc462eac --- /dev/null +++ b/crates/controller-admin-datasets/src/ctx.rs @@ -0,0 +1,18 @@ +use amp_data_store::DataStore; +use amp_datasets_registry::DatasetsRegistry; +use common::{datasets_cache::DatasetsCache, udfs::eth_call::EthCallUdfsCache}; +use metadata_db::MetadataDb; + +/// The controller-admin-datasets context +#[derive(Clone)] +pub struct Ctx { + pub metadata_db: MetadataDb, + /// Datasets registry for manifest and version tag operations. + pub datasets_registry: DatasetsRegistry, + /// Datasets cache for loading datasets. + pub datasets_cache: DatasetsCache, + /// EthCall UDFs cache for eth_call UDF creation. + pub ethcall_udfs_cache: EthCallUdfsCache, + /// Object store for output data. + pub data_store: DataStore, +} diff --git a/crates/controller-admin-datasets/src/datasets.rs b/crates/controller-admin-datasets/src/datasets.rs new file mode 100644 index 000000000..c3d449565 --- /dev/null +++ b/crates/controller-admin-datasets/src/datasets.rs @@ -0,0 +1 @@ +pub mod handlers; diff --git a/crates/controller-admin-datasets/src/datasets/handlers.rs b/crates/controller-admin-datasets/src/datasets/handlers.rs new file mode 100644 index 000000000..3207442a9 --- /dev/null +++ b/crates/controller-admin-datasets/src/datasets/handlers.rs @@ -0,0 +1,8 @@ +pub mod delete; +pub mod delete_version; +pub mod get; +pub mod get_all; +pub mod get_manifest; +pub mod list_all; +pub mod list_versions; +pub mod register; diff --git a/crates/services/admin-api/src/handlers/datasets/delete.rs b/crates/controller-admin-datasets/src/datasets/handlers/delete.rs similarity index 99% rename from crates/services/admin-api/src/handlers/datasets/delete.rs rename to crates/controller-admin-datasets/src/datasets/handlers/delete.rs index 343be55ba..c729e5c72 100644 --- a/crates/services/admin-api/src/handlers/datasets/delete.rs +++ b/crates/controller-admin-datasets/src/datasets/handlers/delete.rs @@ -9,7 +9,7 @@ use monitoring::logging; use crate::{ ctx::Ctx, - handlers::error::{ErrorResponse, IntoErrorResponse}, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `DELETE /datasets/{namespace}/{name}` endpoint diff --git a/crates/services/admin-api/src/handlers/datasets/delete_version.rs b/crates/controller-admin-datasets/src/datasets/handlers/delete_version.rs similarity index 99% rename from crates/services/admin-api/src/handlers/datasets/delete_version.rs rename to crates/controller-admin-datasets/src/datasets/handlers/delete_version.rs index ad91bf901..8aea3221b 100644 --- a/crates/services/admin-api/src/handlers/datasets/delete_version.rs +++ b/crates/controller-admin-datasets/src/datasets/handlers/delete_version.rs @@ -8,7 +8,7 @@ use monitoring::logging; use crate::{ ctx::Ctx, - handlers::error::{ErrorResponse, IntoErrorResponse}, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `DELETE /datasets/{namespace}/{name}/versions/{version}` endpoint diff --git a/crates/services/admin-api/src/handlers/datasets/get.rs b/crates/controller-admin-datasets/src/datasets/handlers/get.rs similarity index 99% rename from crates/services/admin-api/src/handlers/datasets/get.rs rename to crates/controller-admin-datasets/src/datasets/handlers/get.rs index 0ad8d6dbc..8c512a5e8 100644 --- a/crates/services/admin-api/src/handlers/datasets/get.rs +++ b/crates/controller-admin-datasets/src/datasets/handlers/get.rs @@ -13,7 +13,7 @@ use monitoring::logging; use crate::{ ctx::Ctx, - handlers::error::{ErrorResponse, IntoErrorResponse}, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `GET /datasets/{namespace}/{name}/versions/{revision}` endpoint diff --git a/crates/services/admin-api/src/handlers/datasets/get_all.rs b/crates/controller-admin-datasets/src/datasets/handlers/get_all.rs similarity index 97% rename from crates/services/admin-api/src/handlers/datasets/get_all.rs rename to crates/controller-admin-datasets/src/datasets/handlers/get_all.rs index 92c837ab6..83977811a 100644 --- a/crates/services/admin-api/src/handlers/datasets/get_all.rs +++ b/crates/controller-admin-datasets/src/datasets/handlers/get_all.rs @@ -6,7 +6,7 @@ use monitoring::logging; use crate::{ ctx::Ctx, - handlers::error::{ErrorResponse, IntoErrorResponse}, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `GET /datasets` endpoint @@ -38,7 +38,7 @@ use crate::{ operation_id = "datasets_list", responses( (status = 200, description = "Returns all datasets", body = DatasetsResponse), - (status = 500, description = "Internal server error", body = crate::handlers::error::ErrorResponse) + (status = 500, description = "Internal server error", body = ErrorResponse) ) ) )] diff --git a/crates/services/admin-api/src/handlers/datasets/get_manifest.rs b/crates/controller-admin-datasets/src/datasets/handlers/get_manifest.rs similarity index 99% rename from crates/services/admin-api/src/handlers/datasets/get_manifest.rs rename to crates/controller-admin-datasets/src/datasets/handlers/get_manifest.rs index 5a9432ef3..88c200512 100644 --- a/crates/services/admin-api/src/handlers/datasets/get_manifest.rs +++ b/crates/controller-admin-datasets/src/datasets/handlers/get_manifest.rs @@ -13,7 +13,7 @@ use serde_json::Value as JsonValue; use crate::{ ctx::Ctx, - handlers::error::{ErrorResponse, IntoErrorResponse}, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `GET /datasets/{namespace}/{name}/versions/{revision}/manifest` endpoint diff --git a/crates/services/admin-api/src/handlers/datasets/list_all.rs b/crates/controller-admin-datasets/src/datasets/handlers/list_all.rs similarity index 98% rename from crates/services/admin-api/src/handlers/datasets/list_all.rs rename to crates/controller-admin-datasets/src/datasets/handlers/list_all.rs index 5361b9dd2..84ef118b3 100644 --- a/crates/services/admin-api/src/handlers/datasets/list_all.rs +++ b/crates/controller-admin-datasets/src/datasets/handlers/list_all.rs @@ -4,7 +4,7 @@ use datasets_common::{name::Name, namespace::Namespace, version::Version}; use crate::{ ctx::Ctx, - handlers::error::{ErrorResponse, IntoErrorResponse}, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `GET /datasets` endpoint diff --git a/crates/services/admin-api/src/handlers/datasets/list_versions.rs b/crates/controller-admin-datasets/src/datasets/handlers/list_versions.rs similarity index 99% rename from crates/services/admin-api/src/handlers/datasets/list_versions.rs rename to crates/controller-admin-datasets/src/datasets/handlers/list_versions.rs index 6f1a0a5de..8862193ac 100644 --- a/crates/services/admin-api/src/handlers/datasets/list_versions.rs +++ b/crates/controller-admin-datasets/src/datasets/handlers/list_versions.rs @@ -9,7 +9,7 @@ use monitoring::logging; use crate::{ ctx::Ctx, - handlers::error::{ErrorResponse, IntoErrorResponse}, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `GET /datasets/{namespace}/{name}/versions` endpoint diff --git a/crates/services/admin-api/src/handlers/datasets/register.rs b/crates/controller-admin-datasets/src/datasets/handlers/register.rs similarity index 98% rename from crates/services/admin-api/src/handlers/datasets/register.rs rename to crates/controller-admin-datasets/src/datasets/handlers/register.rs index daba2b724..47a2be8b5 100644 --- a/crates/services/admin-api/src/handlers/datasets/register.rs +++ b/crates/controller-admin-datasets/src/datasets/handlers/register.rs @@ -19,15 +19,13 @@ use monitoring::logging; use serde_json::value::RawValue; use crate::{ - ctx::Ctx, - handlers::{ - common::{ - DatasetKind, ManifestHeader, ManifestValidationError, ParseDerivedManifestError, - ParseRawManifestError, parse_and_canonicalize_derived_dataset_manifest, - parse_and_canonicalize_raw_dataset_manifest, - }, - error::{ErrorResponse, IntoErrorResponse}, + common::{ + DatasetKind, ManifestHeader, ManifestValidationError, ParseDerivedManifestError, + ParseRawManifestError, parse_and_canonicalize_derived_dataset_manifest, + parse_and_canonicalize_raw_dataset_manifest, }, + ctx::Ctx, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `POST /datasets` endpoint @@ -113,8 +111,8 @@ use crate::{ request_body = RegisterRequest, responses( (status = 201, description = "Dataset successfully registered or updated", body = RegisterResponse), - (status = 400, description = "Invalid request format or manifest", body = crate::handlers::error::ErrorResponse), - (status = 500, description = "Internal server error", body = crate::handlers::error::ErrorResponse) + (status = 400, description = "Invalid request format or manifest", body = ErrorResponse), + (status = 500, description = "Internal server error", body = ErrorResponse) ) ) )] diff --git a/crates/controller-admin-datasets/src/error.rs b/crates/controller-admin-datasets/src/error.rs new file mode 100644 index 000000000..983ccfb95 --- /dev/null +++ b/crates/controller-admin-datasets/src/error.rs @@ -0,0 +1,95 @@ +//! Error handling types for HTTP handlers + +use std::error::Error as StdError; + +use axum::{Json, http::StatusCode}; + +/// Standard error response returned by the API +/// +/// This struct represents error information returned in HTTP error responses. +/// It provides structured error details including a machine-readable error code +/// and human-readable message. +/// +/// ## Error Code Conventions +/// - Error codes use SCREAMING_SNAKE_CASE (e.g., `DATASET_NOT_FOUND`) +/// - Codes are stable and can be relied upon programmatically +/// - Messages may change and should only be used for display/logging +/// +/// ## Example JSON Response +/// ```json +/// { +/// "error_code": "DATASET_NOT_FOUND", +/// "error_message": "dataset 'eth_mainnet' version '1.0.0' not found" +/// } +/// ``` +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +pub struct ErrorResponse { + /// HTTP status code for this error + /// + /// Not serialized to JSON - used internally for response construction + #[serde(skip)] + pub status_code: StatusCode, + + /// Machine-readable error code in SCREAMING_SNAKE_CASE format + /// + /// Error codes are stable across API versions and should be used + /// for programmatic error handling. Examples: `INVALID_SELECTOR`, + /// `DATASET_NOT_FOUND`, `METADATA_DB_ERROR` + pub error_code: String, + + /// Human-readable error message + /// + /// Messages provide detailed context about the error but may change + /// over time. Use `error_code` for programmatic decisions. + pub error_message: String, +} + +/// Trait for error types that can be converted to HTTP error responses +/// +/// This trait must be implemented by all handler-specific error enums to enable +/// automatic conversion into `ErrorResponse`. +pub trait IntoErrorResponse: StdError + Send + Sync + 'static { + /// Returns a stable, machine-readable error code + /// + /// Error codes should use SCREAMING_SNAKE_CASE and remain stable across versions. + fn error_code(&self) -> &'static str; + + /// Returns the HTTP status code for this error + fn status_code(&self) -> StatusCode; +} + +impl From for ErrorResponse +where + E: IntoErrorResponse, +{ + fn from(error: E) -> Self { + // Format error message with source chain + let mut message = error.to_string(); + let mut current = error.source(); + + while let Some(source) = current { + message.push_str(": "); + message.push_str(&source.to_string()); + current = source.source(); + } + + ErrorResponse { + status_code: error.status_code(), + error_code: error.error_code().to_string(), + error_message: message, + } + } +} + +impl std::fmt::Display for ErrorResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.error_message.fmt(f) + } +} + +impl axum::response::IntoResponse for ErrorResponse { + fn into_response(self) -> axum::response::Response { + (self.status_code, Json(self)).into_response() + } +} diff --git a/crates/controller-admin-datasets/src/lib.rs b/crates/controller-admin-datasets/src/lib.rs new file mode 100644 index 000000000..7989d10e5 --- /dev/null +++ b/crates/controller-admin-datasets/src/lib.rs @@ -0,0 +1,55 @@ +use axum::{ + Router, + routing::{get, post}, +}; + +pub mod common; +pub mod ctx; +pub mod datasets; +pub mod error; +pub mod manifests; +pub mod schema; + +use self::{ + ctx::Ctx, datasets::handlers as dataset_handlers, manifests::handlers as manifest_handlers, +}; + +pub fn router() -> Router { + Router::new() + .route( + "/datasets", + get(dataset_handlers::list_all::handler).post(dataset_handlers::register::handler), + ) + .route( + "/datasets/{namespace}/{name}", + get(dataset_handlers::get::handler).delete(dataset_handlers::delete::handler), + ) + .route( + "/datasets/{namespace}/{name}/versions", + get(dataset_handlers::list_versions::handler), + ) + .route( + "/datasets/{namespace}/{name}/versions/{version}", + get(dataset_handlers::get::handler).delete(dataset_handlers::delete_version::handler), + ) + .route( + "/datasets/{namespace}/{name}/versions/{revision}/manifest", + get(dataset_handlers::get_manifest::handler), + ) + .route( + "/manifests", + get(manifest_handlers::list_all::handler) + .post(manifest_handlers::register::handler) + .delete(manifest_handlers::prune::handler), + ) + .route( + "/manifests/{hash}", + get(manifest_handlers::get_by_id::handler) + .delete(manifest_handlers::delete_by_id::handler), + ) + .route( + "/manifests/{hash}/datasets", + get(manifest_handlers::list_datasets::handler), + ) + .route("/schema", post(schema::handler)) +} diff --git a/crates/controller-admin-datasets/src/manifests.rs b/crates/controller-admin-datasets/src/manifests.rs new file mode 100644 index 000000000..c3d449565 --- /dev/null +++ b/crates/controller-admin-datasets/src/manifests.rs @@ -0,0 +1 @@ +pub mod handlers; diff --git a/crates/services/admin-api/src/handlers/manifests.rs b/crates/controller-admin-datasets/src/manifests/handlers.rs similarity index 100% rename from crates/services/admin-api/src/handlers/manifests.rs rename to crates/controller-admin-datasets/src/manifests/handlers.rs diff --git a/crates/services/admin-api/src/handlers/manifests/delete_by_id.rs b/crates/controller-admin-datasets/src/manifests/handlers/delete_by_id.rs similarity index 98% rename from crates/services/admin-api/src/handlers/manifests/delete_by_id.rs rename to crates/controller-admin-datasets/src/manifests/handlers/delete_by_id.rs index 36b259afa..c1a603f05 100644 --- a/crates/services/admin-api/src/handlers/manifests/delete_by_id.rs +++ b/crates/controller-admin-datasets/src/manifests/handlers/delete_by_id.rs @@ -10,7 +10,7 @@ use monitoring::logging; use crate::{ ctx::Ctx, - handlers::error::{ErrorResponse, IntoErrorResponse}, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `DELETE /manifests/{hash}` endpoint @@ -66,9 +66,9 @@ use crate::{ ), responses( (status = 204, description = "Manifest successfully deleted (or already deleted)"), - (status = 400, description = "Invalid hash", body = crate::handlers::error::ErrorResponse), - (status = 409, description = "Manifest linked to datasets", body = crate::handlers::error::ErrorResponse), - (status = 500, description = "Internal server error", body = crate::handlers::error::ErrorResponse) + (status = 400, description = "Invalid hash", body = ErrorResponse), + (status = 409, description = "Manifest linked to datasets", body = ErrorResponse), + (status = 500, description = "Internal server error", body = ErrorResponse) ) ) )] diff --git a/crates/services/admin-api/src/handlers/manifests/get_by_id.rs b/crates/controller-admin-datasets/src/manifests/handlers/get_by_id.rs similarity index 97% rename from crates/services/admin-api/src/handlers/manifests/get_by_id.rs rename to crates/controller-admin-datasets/src/manifests/handlers/get_by_id.rs index 529c94e1b..07820d842 100644 --- a/crates/services/admin-api/src/handlers/manifests/get_by_id.rs +++ b/crates/controller-admin-datasets/src/manifests/handlers/get_by_id.rs @@ -12,7 +12,7 @@ use serde_json::value::RawValue as JsonRawValue; use crate::{ ctx::Ctx, - handlers::error::{ErrorResponse, IntoErrorResponse}, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `GET /manifests/{hash}` endpoint @@ -56,9 +56,9 @@ use crate::{ ), responses( (status = 200, description = "Successfully retrieved manifest JSON (schema varies by manifest kind)", body = ManifestResponse), - (status = 400, description = "Invalid manifest hash", body = crate::handlers::error::ErrorResponse), - (status = 404, description = "Manifest not found", body = crate::handlers::error::ErrorResponse), - (status = 500, description = "Manifest retrieval error", body = crate::handlers::error::ErrorResponse) + (status = 400, description = "Invalid manifest hash", body = ErrorResponse), + (status = 404, description = "Manifest not found", body = ErrorResponse), + (status = 500, description = "Manifest retrieval error", body = ErrorResponse) ) ) )] diff --git a/crates/services/admin-api/src/handlers/manifests/list_all.rs b/crates/controller-admin-datasets/src/manifests/handlers/list_all.rs similarity index 97% rename from crates/services/admin-api/src/handlers/manifests/list_all.rs rename to crates/controller-admin-datasets/src/manifests/handlers/list_all.rs index 4be197ae0..1af10ecb2 100644 --- a/crates/services/admin-api/src/handlers/manifests/list_all.rs +++ b/crates/controller-admin-datasets/src/manifests/handlers/list_all.rs @@ -3,7 +3,7 @@ use datasets_common::hash::Hash; use crate::{ ctx::Ctx, - handlers::error::{ErrorResponse, IntoErrorResponse}, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `GET /manifests` endpoint @@ -35,7 +35,7 @@ use crate::{ operation_id = "list_all_manifests", responses( (status = 200, description = "Successfully retrieved all manifests", body = ManifestsResponse), - (status = 500, description = "Internal server error", body = crate::handlers::error::ErrorResponse) + (status = 500, description = "Internal server error", body = ErrorResponse) ) ) )] diff --git a/crates/services/admin-api/src/handlers/manifests/list_datasets.rs b/crates/controller-admin-datasets/src/manifests/handlers/list_datasets.rs similarity index 99% rename from crates/services/admin-api/src/handlers/manifests/list_datasets.rs rename to crates/controller-admin-datasets/src/manifests/handlers/list_datasets.rs index b4ed06cb9..d7e635900 100644 --- a/crates/services/admin-api/src/handlers/manifests/list_datasets.rs +++ b/crates/controller-admin-datasets/src/manifests/handlers/list_datasets.rs @@ -9,7 +9,7 @@ use monitoring::logging; use crate::{ ctx::Ctx, - handlers::error::{ErrorResponse, IntoErrorResponse}, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `GET /manifests/{hash}/datasets` endpoint diff --git a/crates/services/admin-api/src/handlers/manifests/prune.rs b/crates/controller-admin-datasets/src/manifests/handlers/prune.rs similarity index 98% rename from crates/services/admin-api/src/handlers/manifests/prune.rs rename to crates/controller-admin-datasets/src/manifests/handlers/prune.rs index 7ece7e316..b38aa0df6 100644 --- a/crates/services/admin-api/src/handlers/manifests/prune.rs +++ b/crates/controller-admin-datasets/src/manifests/handlers/prune.rs @@ -7,7 +7,7 @@ use monitoring::logging; use crate::{ ctx::Ctx, - handlers::error::{ErrorResponse, IntoErrorResponse}, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `DELETE /manifests` endpoint @@ -42,7 +42,7 @@ use crate::{ operation_id = "manifests_prune", responses( (status = 200, description = "Orphaned manifests pruned successfully", body = PruneResponse), - (status = 500, description = "Internal server error", body = crate::handlers::error::ErrorResponse) + (status = 500, description = "Internal server error", body = ErrorResponse) ) ) )] diff --git a/crates/services/admin-api/src/handlers/manifests/register.rs b/crates/controller-admin-datasets/src/manifests/handlers/register.rs similarity index 95% rename from crates/services/admin-api/src/handlers/manifests/register.rs rename to crates/controller-admin-datasets/src/manifests/handlers/register.rs index 2ffe9a00c..263aa8152 100644 --- a/crates/services/admin-api/src/handlers/manifests/register.rs +++ b/crates/controller-admin-datasets/src/manifests/handlers/register.rs @@ -14,15 +14,13 @@ use datasets_common::{ use monitoring::logging; use crate::{ - ctx::Ctx, - handlers::{ - common::{ - DatasetKind, ManifestHeader, ManifestValidationError, ParseDerivedManifestError, - ParseRawManifestError, parse_and_canonicalize_derived_dataset_manifest, - parse_and_canonicalize_raw_dataset_manifest, - }, - error::{ErrorResponse, IntoErrorResponse}, + common::{ + DatasetKind, ManifestHeader, ManifestValidationError, ParseDerivedManifestError, + ParseRawManifestError, parse_and_canonicalize_derived_dataset_manifest, + parse_and_canonicalize_raw_dataset_manifest, }, + ctx::Ctx, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `POST /manifests` endpoint @@ -71,8 +69,8 @@ use crate::{ request_body = serde_json::Value, responses( (status = 201, description = "Manifest successfully registered", body = RegisterManifestResponse), - (status = 400, description = "Invalid request format or manifest", body = crate::handlers::error::ErrorResponse), - (status = 500, description = "Internal server error", body = crate::handlers::error::ErrorResponse) + (status = 400, description = "Invalid request format or manifest", body = ErrorResponse), + (status = 500, description = "Internal server error", body = ErrorResponse) ) ) )] diff --git a/crates/services/admin-api/src/handlers/schema.rs b/crates/controller-admin-datasets/src/schema.rs similarity index 98% rename from crates/services/admin-api/src/handlers/schema.rs rename to crates/controller-admin-datasets/src/schema.rs index 46e621b6d..4a6dc1991 100644 --- a/crates/services/admin-api/src/handlers/schema.rs +++ b/crates/controller-admin-datasets/src/schema.rs @@ -29,11 +29,9 @@ use datasets_derived::{ }; use crate::{ + common::{InterTableDepError, resolve_inter_table_order}, ctx::Ctx, - handlers::{ - common::{InterTableDepError, resolve_inter_table_order}, - error::{ErrorResponse, IntoErrorResponse}, - }, + error::{ErrorResponse, IntoErrorResponse}, }; /// Handler for the `POST /schema` endpoint @@ -106,9 +104,9 @@ use crate::{ request_body = SchemaRequest, responses( (status = 200, description = "Successfully analyzed SQL query and returned schema", body = SchemaResponse), - (status = 400, description = "Client error: Invalid SQL, table references, or function syntax", body = crate::handlers::error::ErrorResponse), - (status = 404, description = "Dataset not found", body = crate::handlers::error::ErrorResponse), - (status = 500, description = "Server error: Dataset store, planning, or internal failures", body = crate::handlers::error::ErrorResponse) + (status = 400, description = "Client error: Invalid SQL, table references, or function syntax", body = ErrorResponse), + (status = 404, description = "Dataset not found", body = ErrorResponse), + (status = 500, description = "Server error: Dataset store, planning, or internal failures", body = ErrorResponse) ) ) )] diff --git a/crates/controller-admin-tables/src/datasets.rs b/crates/controller-admin-tables/src/datasets.rs new file mode 100644 index 000000000..c3d449565 --- /dev/null +++ b/crates/controller-admin-tables/src/datasets.rs @@ -0,0 +1 @@ +pub mod handlers; diff --git a/crates/controller-admin-tables/src/datasets/handlers.rs b/crates/controller-admin-tables/src/datasets/handlers.rs new file mode 100644 index 000000000..07d015b90 --- /dev/null +++ b/crates/controller-admin-tables/src/datasets/handlers.rs @@ -0,0 +1,2 @@ +pub mod restore; +pub mod restore_table; diff --git a/crates/services/admin-api/src/handlers/datasets/restore.rs b/crates/controller-admin-tables/src/datasets/handlers/restore.rs similarity index 98% rename from crates/services/admin-api/src/handlers/datasets/restore.rs rename to crates/controller-admin-tables/src/datasets/handlers/restore.rs index 89a0b1cda..171dddd4b 100644 --- a/crates/services/admin-api/src/handlers/datasets/restore.rs +++ b/crates/controller-admin-tables/src/datasets/handlers/restore.rs @@ -16,10 +16,8 @@ use tokio::task::JoinHandle; use crate::{ ctx::Ctx, - handlers::{ - common::{RegisterRevisionFilesError, register_revision_files}, - error::{ErrorResponse, IntoErrorResponse}, - }, + error::{ErrorResponse, IntoErrorResponse}, + revisions::register_files::{RegisterRevisionFilesError, register_revision_files}, }; /// Handler for the `POST /datasets/{namespace}/{name}/versions/{revision}/restore` endpoint diff --git a/crates/services/admin-api/src/handlers/datasets/restore_table.rs b/crates/controller-admin-tables/src/datasets/handlers/restore_table.rs similarity index 99% rename from crates/services/admin-api/src/handlers/datasets/restore_table.rs rename to crates/controller-admin-tables/src/datasets/handlers/restore_table.rs index ff105532d..8e62def58 100644 --- a/crates/services/admin-api/src/handlers/datasets/restore_table.rs +++ b/crates/controller-admin-tables/src/datasets/handlers/restore_table.rs @@ -13,10 +13,8 @@ use monitoring::logging; use crate::{ ctx::Ctx, - handlers::{ - common::{RegisterRevisionFilesError, register_revision_files}, - error::{ErrorResponse, IntoErrorResponse}, - }, + error::{ErrorResponse, IntoErrorResponse}, + revisions::register_files::{RegisterRevisionFilesError, register_revision_files}, }; /// Handler for the `POST /datasets/{namespace}/{name}/versions/{revision}/tables/{table_name}/restore` endpoint diff --git a/crates/controller-admin-tables/src/lib.rs b/crates/controller-admin-tables/src/lib.rs index fdf19246a..a75805a32 100644 --- a/crates/controller-admin-tables/src/lib.rs +++ b/crates/controller-admin-tables/src/lib.rs @@ -1,15 +1,18 @@ +use axum::{ + Router, + routing::{delete, get, post}, +}; + pub mod ctx; +pub mod datasets; pub mod error; pub mod files; pub mod revisions; -use axum::{ - Router, - routing::{delete, get, post}, +use self::{ + ctx::Ctx, datasets::handlers as dataset_handlers, files::handlers as file_handlers, + revisions::handlers as revision_handlers, }; -use ctx::Ctx; -use files::handlers as file_handlers; -use revisions::handlers as revision_handlers; pub fn router() -> Router { Router::new() @@ -42,4 +45,12 @@ pub fn router() -> Router { post(revision_handlers::deactivate::handler), ) .route("/files/{file_id}", get(file_handlers::get_by_id::handler)) + .route( + "/datasets/{namespace}/{name}/versions/{revision}/restore", + post(dataset_handlers::restore::handler), + ) + .route( + "/datasets/{namespace}/{name}/versions/{revision}/tables/{table_name}/restore", + post(dataset_handlers::restore_table::handler), + ) } diff --git a/crates/controller-admin-tables/src/revisions.rs b/crates/controller-admin-tables/src/revisions.rs index 4440e7349..5514b2375 100644 --- a/crates/controller-admin-tables/src/revisions.rs +++ b/crates/controller-admin-tables/src/revisions.rs @@ -1,4 +1,4 @@ pub mod handlers; pub mod revision_guard; -mod register_files; +pub(crate) mod register_files; diff --git a/crates/core/monitoring/src/logging.rs b/crates/core/monitoring/src/logging.rs index 5870ee690..81bd65d53 100644 --- a/crates/core/monitoring/src/logging.rs +++ b/crates/core/monitoring/src/logging.rs @@ -67,6 +67,7 @@ const AMP_CRATES: &[&str] = &[ "admin_client", "amp_client", "amp_config", + "amp_controller_admin_datasets", "amp_controller_admin_system", "amp_controller_admin_tables", "amp_data_store", diff --git a/crates/services/admin-api/Cargo.toml b/crates/services/admin-api/Cargo.toml index 2c0973274..c61a18889 100644 --- a/crates/services/admin-api/Cargo.toml +++ b/crates/services/admin-api/Cargo.toml @@ -5,15 +5,14 @@ license-file.workspace = true edition.workspace = true [features] -utoipa = ["dep:utoipa", "amp-controller-admin-system/utoipa", "amp-controller-admin-tables/utoipa"] +utoipa = ["dep:utoipa", "amp-controller-admin-datasets/utoipa", "amp-controller-admin-system/utoipa", "amp-controller-admin-tables/utoipa"] [dependencies] +amp-controller-admin-datasets = { path = "../../controller-admin-datasets" } amp-controller-admin-system = { path = "../../controller-admin-system" } amp-controller-admin-tables = { path = "../../controller-admin-tables" } amp-data-store = { path = "../../core/data-store" } -amp-datasets-raw = { path = "../../core/datasets-raw", package = "datasets-raw" } amp-datasets-registry = { path = "../../core/datasets-registry" } -amp-parquet = { path = "../../core/parquet" } amp-providers-common = { path = "../../core/providers-common" } amp-providers-registry = { path = "../../core/providers-registry" } amp-worker-core = { path = "../../core/worker-core" } @@ -23,16 +22,13 @@ amp-worker-gc = { path = "../../core/worker-gc" } async-trait.workspace = true axum.workspace = true common = { path = "../../core/common" } -datafusion.workspace = true datasets-common = { path = "../../core/datasets-common" } datasets-derived = { path = "../../core/datasets-derived" } -futures.workspace = true metadata-db = { path = "../../core/metadata-db" } monitoring = { path = "../../core/monitoring" } serde.workspace = true serde_json = { workspace = true, features = ["raw_value"] } thiserror.workspace = true -tokio.workspace = true toml.workspace = true tracing.workspace = true utoipa = { version = "5.4.0", features = ["axum_extras"], optional = true } diff --git a/crates/services/admin-api/src/handlers.rs b/crates/services/admin-api/src/handlers.rs index 5171429cd..0d80e92ba 100644 --- a/crates/services/admin-api/src/handlers.rs +++ b/crates/services/admin-api/src/handlers.rs @@ -1,8 +1,5 @@ -pub mod common; pub mod error; pub mod datasets; pub mod jobs; -pub mod manifests; pub mod providers; -pub mod schema; diff --git a/crates/services/admin-api/src/handlers/datasets.rs b/crates/services/admin-api/src/handlers/datasets.rs index 0f284f560..39235920b 100644 --- a/crates/services/admin-api/src/handlers/datasets.rs +++ b/crates/services/admin-api/src/handlers/datasets.rs @@ -1,14 +1,4 @@ //! Datasets HTTP handlers -pub mod delete; -pub mod delete_version; pub mod deploy; -pub mod get; -pub mod get_all; -pub mod get_manifest; -pub mod list_all; pub mod list_jobs; -pub mod list_versions; -pub mod register; -pub mod restore; -pub mod restore_table; diff --git a/crates/services/admin-api/src/lib.rs b/crates/services/admin-api/src/lib.rs index 911426c38..503cfab68 100644 --- a/crates/services/admin-api/src/lib.rs +++ b/crates/services/admin-api/src/lib.rs @@ -13,7 +13,7 @@ pub mod handlers; pub mod scheduler; use ctx::Ctx; -use handlers::{datasets, jobs, manifests, providers, schema}; +use handlers::{datasets, jobs, providers}; use crate::ctx::{RevisionGuardImpl, WorkerServiceImpl}; @@ -31,40 +31,19 @@ pub fn router(ctx: Ctx) -> Router<()> { let system_ctx = amp_controller_admin_system::ctx::Ctx { worker_service: Arc::new(WorkerServiceImpl(ctx.scheduler.clone())), }; + let datasets_ctx = amp_controller_admin_datasets::ctx::Ctx { + metadata_db: ctx.metadata_db.clone(), + datasets_registry: ctx.datasets_registry.clone(), + datasets_cache: ctx.datasets_cache.clone(), + ethcall_udfs_cache: ctx.ethcall_udfs_cache.clone(), + data_store: ctx.data_store.clone(), + }; Router::new() - .route( - "/datasets", - get(datasets::list_all::handler).post(datasets::register::handler), - ) - .route( - "/datasets/{namespace}/{name}", - get(datasets::get::handler).delete(datasets::delete::handler), - ) - .route( - "/datasets/{namespace}/{name}/versions", - get(datasets::list_versions::handler), - ) - .route( - "/datasets/{namespace}/{name}/versions/{version}", - get(datasets::get::handler).delete(datasets::delete_version::handler), - ) - .route( - "/datasets/{namespace}/{name}/versions/{revision}/manifest", - get(datasets::get_manifest::handler), - ) .route( "/datasets/{namespace}/{name}/versions/{revision}/deploy", post(datasets::deploy::handler), ) - .route( - "/datasets/{namespace}/{name}/versions/{revision}/restore", - post(datasets::restore::handler), - ) - .route( - "/datasets/{namespace}/{name}/versions/{revision}/tables/{table_name}/restore", - post(datasets::restore_table::handler), - ) .route( "/datasets/{namespace}/{name}/versions/{revision}/jobs", get(datasets::list_jobs::handler), @@ -86,20 +65,6 @@ pub fn router(ctx: Ctx) -> Router<()> { "/jobs/{id}/events/{event_id}", get(jobs::event_by_id::handler), ) - .route( - "/manifests", - get(manifests::list_all::handler) - .post(manifests::register::handler) - .delete(manifests::prune::handler), - ) - .route( - "/manifests/{hash}", - get(manifests::get_by_id::handler).delete(manifests::delete_by_id::handler), - ) - .route( - "/manifests/{hash}/datasets", - get(manifests::list_datasets::handler), - ) .route( "/providers", get(providers::get_all::handler).post(providers::create::handler), @@ -108,8 +73,8 @@ pub fn router(ctx: Ctx) -> Router<()> { "/providers/{name}", get(providers::get_by_id::handler).delete(providers::delete_by_id::handler), ) - .route("/schema", post(schema::handler)) .with_state(ctx) + .merge(amp_controller_admin_datasets::router().with_state(datasets_ctx)) .merge(amp_controller_admin_system::router().with_state(system_ctx)) .merge(amp_controller_admin_tables::router().with_state(tables_ctx)) } @@ -124,24 +89,24 @@ pub fn router(ctx: Ctx) -> Router<()> { ), paths( // Dataset endpoints - handlers::datasets::list_all::handler, - handlers::datasets::list_versions::handler, + amp_controller_admin_datasets::datasets::handlers::list_all::handler, + amp_controller_admin_datasets::datasets::handlers::list_versions::handler, handlers::datasets::list_jobs::handler, - handlers::datasets::get::handler, - handlers::datasets::get_manifest::handler, - handlers::datasets::register::handler, + amp_controller_admin_datasets::datasets::handlers::get::handler, + amp_controller_admin_datasets::datasets::handlers::get_manifest::handler, + amp_controller_admin_datasets::datasets::handlers::register::handler, handlers::datasets::deploy::handler, - handlers::datasets::restore::handler, - handlers::datasets::restore_table::handler, - handlers::datasets::delete::handler, - handlers::datasets::delete_version::handler, + amp_controller_admin_tables::datasets::handlers::restore::handler, + amp_controller_admin_tables::datasets::handlers::restore_table::handler, + amp_controller_admin_datasets::datasets::handlers::delete::handler, + amp_controller_admin_datasets::datasets::handlers::delete_version::handler, // Manifest endpoints - handlers::manifests::list_all::handler, - handlers::manifests::register::handler, - handlers::manifests::get_by_id::handler, - handlers::manifests::delete_by_id::handler, - handlers::manifests::list_datasets::handler, - handlers::manifests::prune::handler, + amp_controller_admin_datasets::manifests::handlers::list_all::handler, + amp_controller_admin_datasets::manifests::handlers::register::handler, + amp_controller_admin_datasets::manifests::handlers::get_by_id::handler, + amp_controller_admin_datasets::manifests::handlers::delete_by_id::handler, + amp_controller_admin_datasets::manifests::handlers::list_datasets::handler, + amp_controller_admin_datasets::manifests::handlers::prune::handler, // Job endpoints handlers::jobs::get_all::handler, handlers::jobs::get_by_id::handler, @@ -159,7 +124,7 @@ pub fn router(ctx: Ctx) -> Router<()> { // Files endpoints amp_controller_admin_tables::files::handlers::get_by_id::handler, // Schema endpoints - handlers::schema::handler, + amp_controller_admin_datasets::schema::handler, // Revision endpoints amp_controller_admin_tables::revisions::handlers::list::handler, amp_controller_admin_tables::revisions::handlers::restore::handler, @@ -178,25 +143,25 @@ pub fn router(ctx: Ctx) -> Router<()> { // Common schemas handlers::error::ErrorResponse, // Manifest schemas - handlers::manifests::list_all::ManifestsResponse, - handlers::manifests::list_all::ManifestInfo, - handlers::manifests::register::RegisterManifestResponse, - handlers::manifests::list_datasets::ManifestDatasetsResponse, - handlers::manifests::list_datasets::Dataset, - handlers::manifests::prune::PruneResponse, + amp_controller_admin_datasets::manifests::handlers::list_all::ManifestsResponse, + amp_controller_admin_datasets::manifests::handlers::list_all::ManifestInfo, + amp_controller_admin_datasets::manifests::handlers::register::RegisterManifestResponse, + amp_controller_admin_datasets::manifests::handlers::list_datasets::ManifestDatasetsResponse, + amp_controller_admin_datasets::manifests::handlers::list_datasets::Dataset, + amp_controller_admin_datasets::manifests::handlers::prune::PruneResponse, // Dataset schemas - handlers::datasets::get::DatasetInfo, - handlers::datasets::list_all::DatasetsResponse, - handlers::datasets::list_all::DatasetSummary, - handlers::datasets::list_versions::VersionsResponse, - handlers::datasets::list_versions::VersionInfo, - handlers::datasets::register::RegisterRequest, - handlers::datasets::register::RegisterResponse, + amp_controller_admin_datasets::datasets::handlers::get::DatasetInfo, + amp_controller_admin_datasets::datasets::handlers::list_all::DatasetsResponse, + amp_controller_admin_datasets::datasets::handlers::list_all::DatasetSummary, + amp_controller_admin_datasets::datasets::handlers::list_versions::VersionsResponse, + amp_controller_admin_datasets::datasets::handlers::list_versions::VersionInfo, + amp_controller_admin_datasets::datasets::handlers::register::RegisterRequest, + amp_controller_admin_datasets::datasets::handlers::register::RegisterResponse, handlers::datasets::deploy::DeployRequest, handlers::datasets::deploy::DeployResponse, - handlers::datasets::restore::RestoreResponse, - handlers::datasets::restore::RestoredTableInfo, - handlers::datasets::restore_table::RestoreTablePayload, + amp_controller_admin_tables::datasets::handlers::restore::RestoreResponse, + amp_controller_admin_tables::datasets::handlers::restore::RestoredTableInfo, + amp_controller_admin_tables::datasets::handlers::restore_table::RestoreTablePayload, // Job schemas handlers::jobs::progress::JobProgressResponse, handlers::jobs::progress::TableProgress, @@ -212,8 +177,8 @@ pub fn router(ctx: Ctx) -> Router<()> { // File schemas amp_controller_admin_tables::files::handlers::get_by_id::FileInfo, // Schema schemas - handlers::schema::SchemaRequest, - handlers::schema::SchemaResponse, + amp_controller_admin_datasets::schema::SchemaRequest, + amp_controller_admin_datasets::schema::SchemaResponse, // Revision schemas amp_controller_admin_tables::revisions::handlers::restore::RestoreResponse, amp_controller_admin_tables::revisions::handlers::activate::ActivationPayload,